From e319f86f15f3bf72d9016d5324500730a1f16e25 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Sat, 3 Oct 2026 22:17:09 +0800 Subject: [PATCH] [Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- docs/training/prompt_token_id_logprobs.md | 34 ++-- .../src/protocol/request.rs | 30 +++- .../src/protocol/sampling.rs | 7 +- .../engine-core-client/src/protocol/tensor.rs | 12 ++ .../src/routes/inference/generate/validate.rs | 2 +- rust/src/text/src/lower.rs | 49 ++++-- rust/src/text/src/lower/logprobs.rs | 67 ++++++-- rust/src/text/src/lower/token_ids.rs | 8 - rust/src/text/src/request.rs | 6 +- .../token_in_token_out/test_serving_tokens.py | 24 ++- tests/v1/sample/test_logprobs.py | 155 +++++++++++++----- tests/v1/sample/test_trace_replay_params.py | 4 +- vllm/outputs.py | 6 +- vllm/sampling_params.py | 86 ++++++---- vllm/v1/engine/input_processor.py | 12 ++ vllm/v1/serial_utils.py | 2 +- vllm/v1/worker/gpu/sample/prompt_logprob.py | 24 +-- 17 files changed, 366 insertions(+), 162 deletions(-) diff --git a/docs/training/prompt_token_id_logprobs.md b/docs/training/prompt_token_id_logprobs.md index 940facad4f..6414c00bfe 100644 --- a/docs/training/prompt_token_id_logprobs.md +++ b/docs/training/prompt_token_id_logprobs.md @@ -2,14 +2,14 @@ In on-policy distillation, the student records its top-K candidate tokens at every response position, and the teacher provides the log probabilities of -exactly those candidates as the distillation target. The caller passes the -candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill -over the prompt plus response and gathers their log probabilities on the GPU at -every scored row, starting from `prompt_logprob_start` (typically the prompt -length minus one, so only response positions are scored). The result is a -`[rows, K]` matrix whose row `i` scores prompt token -`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so -it lines up position by position with the student's top-K. +exactly those candidates as the distillation target. The caller passes one +candidate list per scored row as `prompt_logprob_token_ids`; the teacher runs a +single prefill over the prompt plus response and gathers their log +probabilities on the GPU, starting from `prompt_logprob_start` (typically the +prompt length minus one, so only response positions are scored). The result is +a `[rows, K]` matrix whose row `i` scores prompt token +`prompt_logprob_start + i + 1` and whose columns follow that row's candidate +order, so it lines up position by position with the student's top-K. ## Quick start @@ -22,13 +22,13 @@ output = llm.generate( TokensPrompt(prompt_token_ids=prompt_ids + response_ids), SamplingParams( max_tokens=1, - prompt_logprob_token_ids=candidate_ids, + prompt_logprob_token_ids=student_topk_ids, # [len(response_ids), K] prompt_logprob_start=len(prompt_ids) - 1, ), ) scores = output[0].prompt_token_id_logprobs -# scores.shape == (len(response_ids), len(candidate_ids)) -# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i]) +# scores.shape == (len(response_ids), K) +# scores[i, j] = log p(student_topk_ids[i][j] | prompt_ids + response_ids[:i]) ``` The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint @@ -42,13 +42,17 @@ import numpy as np scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"]))) ``` -The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a -logits `--logprobs-mode`, it holds logits instead of log probabilities. +`prompt_logprob_token_ids` is an integer array or a list of lists with exactly +`prompt_len - 1 - prompt_logprob_start` rows; shorter rows are padded with `-1`, +and every `-1` entry scores `-inf`. Pass an `int32` NumPy array, such as the +student's top-K IDs, which is sent to the engine without conversion; nested +lists of Python ints are much slower to copy and serialize. With a logits +`--logprobs-mode`, the matrix holds logits instead of log probabilities. ## Requirements - The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`). -- `len(prompt_logprob_token_ids)` at most `--max-logprobs`. +- The longest row at most `--max-logprobs` candidates. - No `--kv-sharing-fast-prefill`. The request skips reading the prefix cache (local and KV connector), since @@ -56,8 +60,6 @@ cached rows have no logits to score. ## Limitations -- One candidate set per request, shared by every scored row; per-position - candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)). - Non-streaming only; `stream=true` requests are rejected. - Not exposed through the OpenAI-compatible endpoints. - If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False` diff --git a/rust/src/engine-core-client/src/protocol/request.rs b/rust/src/engine-core-client/src/protocol/request.rs index 8f38d7da08..02fc1c30ca 100644 --- a/rust/src/engine-core-client/src/protocol/request.rs +++ b/rust/src/engine-core-client/src/protocol/request.rs @@ -152,6 +152,11 @@ impl EngineCoreRequest { feature.extract_aux_frames(&mut aux_frames, threshold); } } + if let Some(params) = &mut self.sampling_params + && let Some(ids) = &mut params.prompt_logprob_token_ids + { + ids.extract_aux_frame(&mut aux_frames, threshold); + } aux_frames } } @@ -168,7 +173,7 @@ mod tests { PlaceholderRange, }; use crate::protocol::sampling::EngineCoreSamplingParams; - use crate::protocol::tensor::{WireArrayData, WireTensor}; + use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor}; use crate::protocol::{decode_value, encode_msgpack}; const AUX_FRAME_THRESHOLD: usize = 256; @@ -294,4 +299,27 @@ mod tests { ); assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty()); } + + #[test] + fn engine_core_request_extracts_large_prompt_logprob_token_ids() { + let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD]) + .unwrap(); + let mut request = EngineCoreRequest { + sampling_params: Some(EngineCoreSamplingParams { + prompt_logprob_token_ids: Some(ids), + ..EngineCoreSamplingParams::for_test() + }), + ..EngineCoreRequest::default() + }; + + let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD); + + assert_eq!(aux_frames.len(), 1); + assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4); + let params = request.sampling_params.unwrap(); + assert_eq!( + params.prompt_logprob_token_ids.unwrap().data, + WireArrayData::AuxIndex(1) + ); + } } diff --git a/rust/src/engine-core-client/src/protocol/sampling.rs b/rust/src/engine-core-client/src/protocol/sampling.rs index 0cd11b5397..577dcb651f 100644 --- a/rust/src/engine-core-client/src/protocol/sampling.rs +++ b/rust/src/engine-core-client/src/protocol/sampling.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_default::DefaultFromSerde; use crate::protocol::structured_outputs::StructuredOutputsParams; +use crate::protocol::tensor::WireNdArray; fn default_top_p() -> f32 { 1.0 @@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, - /// Candidate token IDs scored at every scored causal prompt row, where row - /// `i` scores them as predictions of prompt token `i + 1`. - pub prompt_logprob_token_ids: Option>, + /// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row, + /// `-1` padding shorter rows. + pub prompt_logprob_token_ids: Option, /// First causal prompt row to score; `None` scores from the first row. pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. diff --git a/rust/src/engine-core-client/src/protocol/tensor.rs b/rust/src/engine-core-client/src/protocol/tensor.rs index 2bd193f76b..a79c7df5fd 100644 --- a/rust/src/engine-core-client/src/protocol/tensor.rs +++ b/rust/src/engine-core-client/src/protocol/tensor.rs @@ -121,6 +121,18 @@ impl + Serialize + DeserializeOwned> WireArray { )) } + /// Build an int32 tensor/ndarray backed by native-endian raw-view bytes. + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_i32(shape: Vec, data: Vec) -> Result { + validate_element_count(&shape, data.len())?; + Ok(Self::from_raw_bytes( + TensorDtype::I32, + shape, + bytes_from_pod_vec(data), + )) + } + /// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes. /// /// Takes ownership of the backing buffer without copying its data. diff --git a/rust/src/server/src/routes/inference/generate/validate.rs b/rust/src/server/src/routes/inference/generate/validate.rs index 9b1779d6e9..f9e821f945 100644 --- a/rust/src/server/src/routes/inference/generate/validate.rs +++ b/rust/src/server/src/routes/inference/generate/validate.rs @@ -181,7 +181,7 @@ mod tests { let request: GenerateRequest = serde_json::from_value(json!({ "token_ids": [11, 22], "stream": true, - "sampling_params": {"prompt_logprob_token_ids": [1, 2]} + "sampling_params": {"prompt_logprob_token_ids": [[1, 2]]} })) .expect("parse request"); assert!(validate_request_compat(&request, &served(&["test-model"])).is_err()); diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 01a4fb25b8..cf53704374 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -7,7 +7,7 @@ pub(crate) mod logprobs; pub(crate) mod sampling; pub(crate) mod token_ids; -use logprobs::validate_logprobs; +use logprobs::{lower_prompt_logprob_token_ids, validate_logprobs}; use sampling::validate_resolved_sampling_params; use token_ids::{validate_prompt_token_ids, validate_vocab_range}; use vllm_engine_core_client::protocol::sampling::{ @@ -50,7 +50,11 @@ pub fn lower_text_request( // multimodal tensor payloads. mm_features: request.mm_features.take(), sampling_params: lower_sampling_params( - request.sampling_params.clone(), + SamplingParams { + // Move the candidate table rather than cloning it; only lowering reads it. + prompt_logprob_token_ids: request.sampling_params.prompt_logprob_token_ids.take(), + ..request.sampling_params.clone() + }, sampling_hints, sampling_limits, prompt_len, @@ -180,7 +184,12 @@ pub fn lower_sampling_params( thinking_token_budget, logprobs, prompt_logprobs, - prompt_logprob_token_ids, + prompt_logprob_token_ids: lower_prompt_logprob_token_ids( + prompt_logprob_token_ids, + prompt_logprob_start, + prompt_len, + sampling_limits.model_vocab_size, + )?, prompt_logprob_start, min_p, frequency_penalty, @@ -332,6 +341,7 @@ mod tests { use vllm_engine_core_client::protocol::multimodal::{ MmFeatureSpec, MmModality, PlaceholderRange, }; + use vllm_engine_core_client::protocol::tensor::WireNdArray; use vllm_tokenizer::test_utils::TestTokenizer; use super::*; @@ -1140,7 +1150,7 @@ mod tests { #[test] fn lower_sampling_params_validates_prompt_logprob_token_ids() { - let lower = |ids: Option>, start: Option| { + let lower = |ids: Option>>, start: Option| { lower_sampling_params_with_limits( SamplingParams { prompt_logprob_token_ids: ids, @@ -1155,23 +1165,30 @@ mod tests { Ok(_) => panic!("expected rejection"), }; - let params = lower(Some(vec![1, 2]), Some(1)).unwrap(); - assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2])); - assert_eq!(params.prompt_logprob_start, Some(1)); + let params = lower(Some(vec![vec![1, 2], vec![3]]), None).unwrap(); + assert_eq!( + params.prompt_logprob_token_ids, + Some(WireNdArray::from_i32(vec![2, 2], vec![1, 2, 3, -1]).unwrap()) + ); + assert_eq!( + lower(Some(vec![vec![5]]), Some(1)).unwrap().prompt_logprob_start, + Some(1) + ); expect_test::expect![[r#" [ - "prompt_logprob_token_ids must not be empty.", + "prompt_logprob_token_ids must be a non-empty integer array of shape [num_rows, num_ids].", "requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20", - "token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000", - "prompt_logprob_token_ids must not contain duplicates.", + "prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000", + "prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000", + "prompt_logprob_token_ids has 1 rows, but the prompt has 2 scored rows (prompt_len - 1 - prompt_logprob_start).", "prompt_logprob_start requires prompt_logprob_token_ids.", ] - "#]] - .assert_debug_eq(&[ - rejects(Some(vec![]), None), - rejects(Some((0..21).collect()), None), - rejects(Some(vec![1000]), None), - rejects(Some(vec![1, 1]), None), + "#]].assert_debug_eq(&[ + rejects(Some(vec![vec![]]), None), + rejects(Some(vec![(0..21).collect(), vec![0]]), None), + rejects(Some(vec![vec![1000], vec![0]]), None), + rejects(Some(vec![vec![-2], vec![0]]), None), + rejects(Some(vec![vec![1]]), None), rejects(None, Some(0)), ]); } diff --git a/rust/src/text/src/lower/logprobs.rs b/rust/src/text/src/lower/logprobs.rs index 58ad5d694b..407a47c9e1 100644 --- a/rust/src/text/src/lower/logprobs.rs +++ b/rust/src/text/src/lower/logprobs.rs @@ -3,11 +3,11 @@ //! Python-compatible validation for logprobs sampling params. //! -//! `-1` is expanded only for bounds checks. The original request values are -//! passed through to engine-core. +//! `-1` counts are expanded only for bounds checks and passed to engine-core +//! unchanged; `prompt_logprob_token_ids` rows are padded into an array. -use itertools::Itertools as _; use thiserror::Error; +use vllm_engine_core_client::protocol::tensor::WireNdArray; use crate::backend::SamplingLimits; @@ -36,10 +36,21 @@ pub enum LogprobsError { len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}." )] TokenIdsMismatch { logprobs: i32, num_token_ids: usize }, - #[error("prompt_logprob_token_ids must not be empty.")] + #[error( + "prompt_logprob_token_ids must be a non-empty integer array of shape \ + [num_rows, num_ids]." + )] EmptyPromptLogprobTokenIds, - #[error("prompt_logprob_token_ids must not contain duplicates.")] - DuplicatePromptLogprobTokenIds, + #[error( + "prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). \ + Vocabulary size: {vocab_size}" + )] + PromptLogprobTokenIdsOutOfVocab { vocab_size: usize }, + #[error( + "prompt_logprob_token_ids has {rows} rows, but the prompt has {expected} scored \ + rows (prompt_len - 1 - prompt_logprob_start)." + )] + PromptLogprobRowsMismatch { rows: usize, expected: usize }, #[error("prompt_logprob_start requires prompt_logprob_token_ids.")] PromptLogprobStartWithoutTokenIds, } @@ -49,7 +60,7 @@ pub(super) fn validate_logprobs( logprobs: Option, prompt_logprobs: Option, logprob_token_ids: Option<&[u32]>, - prompt_logprob_token_ids: Option<&[u32]>, + prompt_logprob_token_ids: Option<&[Vec]>, prompt_logprob_start: Option, sampling_limits: SamplingLimits, ) -> Result<(), LogprobsError> { @@ -60,7 +71,7 @@ pub(super) fn validate_logprobs( validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?; validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?; validate_logprobs_count( - prompt_logprob_token_ids.map(|ids| ids.len() as i32), + prompt_logprob_token_ids.map(|rows| rows.iter().map(Vec::len).max().unwrap_or(0) as i32), max_logprobs, vocab_size, "prompt_logprob_token_ids", @@ -70,17 +81,49 @@ pub(super) fn validate_logprobs( } fn validate_prompt_logprob_token_ids( - token_ids: Option<&[u32]>, + rows: Option<&[Vec]>, start: Option, ) -> Result<(), LogprobsError> { - match token_ids { - Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds), - Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds), + match rows { + Some(rows) if rows.iter().all(Vec::is_empty) => { + Err(LogprobsError::EmptyPromptLogprobTokenIds) + } None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds), _ => Ok(()), } } +/// Check per-row candidates against the vocabulary and the prompt as Python +/// does, and pad them with `-1` into the engine's `[num_rows, num_ids]` array. +pub(super) fn lower_prompt_logprob_token_ids( + rows: Option>>, + start: Option, + prompt_len: u32, + vocab_size: usize, +) -> Result, LogprobsError> { + let Some(rows) = rows else { + return Ok(None); + }; + if rows.iter().flatten().any(|&id| id < -1 || i64::from(id) >= vocab_size as i64) { + return Err(LogprobsError::PromptLogprobTokenIdsOutOfVocab { vocab_size }); + } + let expected = (prompt_len as usize).saturating_sub(1 + start.unwrap_or(0) as usize); + if rows.len() != expected { + return Err(LogprobsError::PromptLogprobRowsMismatch { + rows: rows.len(), + expected, + }); + } + let width = rows.iter().map(Vec::len).max().unwrap_or(0); + let data = rows + .iter() + .flat_map(|row| row.iter().copied().chain(std::iter::repeat_n(-1, width - row.len()))) + .collect(); + Ok(Some( + WireNdArray::from_i32(vec![rows.len(), width], data).expect("padded rows are rectangular"), + )) +} + fn validate_logprobs_count( requested: Option, max_logprobs: usize, diff --git a/rust/src/text/src/lower/token_ids.rs b/rust/src/text/src/lower/token_ids.rs index 240b0528e1..8512a830e6 100644 --- a/rust/src/text/src/lower/token_ids.rs +++ b/rust/src/text/src/lower/token_ids.rs @@ -95,14 +95,6 @@ pub(crate) fn validate_vocab_range( )?; } - if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() { - validate_param( - "prompt_logprob_token_ids", - token_ids.iter().copied(), - limits.model_vocab_size, - )?; - } - if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() { validate_param( "bad_words", diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 9c9f22a38e..c37e21060e 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -97,9 +97,9 @@ pub struct SamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, - /// Candidate token IDs scored at every scored causal prompt row, where row - /// `i` scores them as predictions of prompt token `i + 1`. - pub prompt_logprob_token_ids: Option>, + /// Candidate token IDs per scored causal prompt row, where row `i` scores + /// its IDs as predictions of prompt token `prompt_logprob_start + i + 1`. + pub prompt_logprob_token_ids: Option>>, /// First causal prompt row to score; `None` scores from the first row. pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. `None` means no diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 4a74448046..2921422f01 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -318,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value): @pytest.mark.asyncio async def test_generate_prompt_token_id_logprobs(client): token_ids = [11, 22, 33, 44, 55] - candidates = [22, 33, 44, 55] + rows = [[33], [44, 100], [55]] sampling_params = { "max_tokens": 1, "prompt_logprobs": 0, - "prompt_logprob_token_ids": candidates, + "prompt_logprob_token_ids": rows, "prompt_logprob_start": 1, } payload = { @@ -335,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client): data = resp.json() scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"]))) - assert scores.shape == (len(token_ids) - 2, len(candidates)) + assert scores.shape == (3, 2) for row, target in enumerate(token_ids[2:]): expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"] - assert scores[row, candidates.index(target)] == pytest.approx( - expected, abs=1e-3 - ) + assert scores[row, 0] == pytest.approx(expected, abs=1e-3) + assert np.isneginf(scores[[0, 2], 1]).all() - stream_only_ids = {"prompt_logprob_token_ids": candidates} + stream_only_ids = {"prompt_logprob_token_ids": rows} resp = await client.post( GEN_ENDPOINT, json={**payload, "sampling_params": stream_only_ids, "stream": True}, @@ -350,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client): assert resp.status_code == 400 assert "prompt_logprob_token_ids" in resp.text - past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)} + too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]} resp = await client.post( - GEN_ENDPOINT, json={**payload, "sampling_params": past_end} + GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows} ) - resp.raise_for_status() - empty = np.load( - io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"])) - ) - assert empty.shape == (0, len(candidates)) + assert resp.status_code == 400 + assert "scored rows" in resp.text @pytest.mark.asyncio diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index 2413c824fc..317ac10d41 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -11,6 +11,7 @@ from unittest.mock import patch import numpy as np import pytest import torch +from pydantic import TypeAdapter, ValidationError from tests.utils import large_gpu_mark from tests.v1.sample.utils import ( @@ -20,7 +21,7 @@ from tests.v1.sample.utils import ( compute_correct_cumulative_logprob, get_test_batch, ) -from vllm import SamplingParams +from vllm import SamplingParams, TokensPrompt from vllm.config.model import LogprobsMode from vllm.distributed import cleanup_dist_env_and_memory from vllm.exceptions import VLLMValidationError @@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]: def _model_config(vocab_size: int = 10, max_logprobs: int = 20): return SimpleNamespace( max_logprobs=max_logprobs, + max_model_len=16, logits_processors=None, is_diffusion=False, get_vocab_size=lambda: vocab_size, @@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]): ) -def test_prompt_logprob_token_ids_bounded_by_max_logprobs(): - """The candidate count is bounded by max_logprobs.""" +def test_prompt_logprob_token_ids_validation(): + """Rows may be ragged (-1 padded); the width is bounded by max_logprobs.""" model_config = _model_config(vocab_size=100, max_logprobs=4) def verify(**kwargs): - SamplingParams(**kwargs).verify( + params = SamplingParams(**kwargs) + params.verify( model_config, speculative_config=None, structured_outputs_config=None, tokenizer=None, ) + return params.prompt_logprob_token_ids - verify(prompt_logprob_token_ids=[1, 2, 3, 4]) + assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [ + [-1] * 4, + [1, -1, -1, -1], + [4, 3, 2, 1], + ] + # Any integer layout is normalized to what the worker uploads: + # C-contiguous native int32. + odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8")) + ids = verify(prompt_logprob_token_ids=odd) + assert ids.dtype == np.int32 and ids.flags.c_contiguous + assert ids.tolist() == [[1, 2], [3, -1]] + # The pydantic path (token-in/token-out API, /docs) takes strict integer lists. + adapter = TypeAdapter(SamplingParams) + for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"): + with pytest.raises(ValidationError): + adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}') + with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"): + verify(prompt_logprob_token_ids=np.array([1, 2])) + with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"): + verify(prompt_logprob_token_ids=[1, 2]) + for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]): + with pytest.raises(VLLMValidationError, match="integer token ids"): + verify(prompt_logprob_token_ids=bad_ids) + with pytest.raises(VLLMValidationError, match="out-of-vocab"): + verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64)) # The error names the value to raise max_logprobs to. with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"): - verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5]) - + verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]]) + with pytest.raises(VLLMValidationError, match="out-of-vocab"): + verify(prompt_logprob_token_ids=[[1], [100]]) + with pytest.raises(VLLMValidationError, match="out-of-vocab"): + verify(prompt_logprob_token_ids=[[1], [-2]]) + with pytest.raises(VLLMValidationError, match="non-empty"): + verify(prompt_logprob_token_ids=[[], []]) + # Rows are bounded by max_model_len before the table is padded. + with pytest.raises(VLLMValidationError, match="at most 15 scored rows"): + verify(prompt_logprob_token_ids=[[]] * 15 + [[1]]) # prompt_logprob_start alone is a caller mistake, not a silent no-op. with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"): verify(prompt_logprob_start=3) @@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner(): tokenizer=None, validate_logits_processors_params=lambda params: None, ) - params = SamplingParams(prompt_logprob_token_ids=[1, 2]) + params = SamplingParams(prompt_logprob_token_ids=[[1, 2]]) with patch.object(SamplingParams, "verify"): with pytest.raises(VLLMValidationError, match="V2 model runner"): InputProcessor._validate_params(processor, params, ("generate",)) @@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption(): def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): - """Fixed-ID scores stay row-aligned across chunked prefill and preemption.""" + """Per-row scores stay row-aligned across chunked prefill and preemption.""" monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") prompts = [ @@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): ] + [f"Tell me about the number {i}: " for i in range(32)] start = 2 - candidate_ids = [10, 100, 1000, 10000] - sampling_params = SamplingParams( - temperature=0.0, - max_tokens=40, - min_tokens=20, - prompt_logprob_token_ids=candidate_ids, - prompt_logprob_start=start, - ) + + def candidate_ids(j): + return [10 + j, 100 + j, 1000 + j, 10000 + j] with VllmRunner( "Qwen/Qwen3-0.6B", @@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): disable_log_stats=False, gpu_memory_utilization=0.25, ) as vllm_model: + tokenizer = vllm_model.llm.get_tokenizer() + token_prompts = [ + TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts + ] + + def make_params(row_ids): + return [ + SamplingParams( + temperature=0.0, + max_tokens=40, + min_tokens=20, + prompt_logprob_token_ids=[ + row_ids(j) + for j in range(len(p["prompt_token_ids"]) - 1 - start) + ], + prompt_logprob_start=start, + ) + for p in token_prompts + ] + metrics_before = vllm_model.llm.get_metrics() - outputs = vllm_model.llm.generate(prompts, sampling_params) + full_params = make_params(candidate_ids) + for params in full_params: # the array form, as a top-k tensor would be + params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids) + outputs = vllm_model.llm.generate(token_prompts, full_params) for i, output in enumerate(outputs): scores = output.prompt_token_id_logprobs - assert scores is not None, f"Output {i} missing fixed-ID scores" + assert scores is not None, f"Output {i} missing per-row scores" expected_shape = ( len(output.prompt_token_ids) - 1 - start, - len(candidate_ids), + 4, ) assert scores.shape == expected_shape, ( f"Output {i} scored {scores.shape}, expected {expected_shape}" @@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): assert math.isfinite(float(scores.min())) assert float(scores.max()) <= 1e-3, "logprobs must be <= 0" + # Ragged rows: each row keeps a prefix of the candidates and must + # reproduce that prefix of the full scores, with -inf padding. + ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4]) + ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params) + for output, ragged_output in zip(outputs, ragged_outputs): + expected = np.full_like(output.prompt_token_id_logprobs, -np.inf) + for j in range(len(expected)): + n = 1 + j % 4 + expected[j, :n] = output.prompt_token_id_logprobs[j, :n] + np.testing.assert_allclose( + ragged_output.prompt_token_id_logprobs, expected, atol=1e-2 + ) + + # The row count is checked against the prompt at admission. + with pytest.raises(VLLMValidationError, match="scored rows"): + vllm_model.llm.generate(token_prompts[0], ragged_params[1]) + metrics_after = vllm_model.llm.get_metrics() preemptions_before = next( (m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0 @@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): # unchunked, unpreempted run of the same requests. Batch composition moves # bf16 tail logprobs by a few percent; a misaligned row differs by nats. with VllmRunner( - "Qwen/Qwen3-0.6B", - max_model_len=512, - max_logprobs=64, - gpu_memory_utilization=0.25, + "Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25 ) as reference: - reference_outputs = reference.llm.generate(prompts, sampling_params) - own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs] + reference_outputs = reference.llm.generate(token_prompts, full_params) self_scored = reference.llm.generate( - prompts, + token_prompts, [ SamplingParams( - max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids + max_tokens=1, + prompt_logprobs=0, + prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]], ) - for ids in own_ids + for p in token_prompts ], ) for output, ref in zip(outputs, reference_outputs): np.testing.assert_allclose( - output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1 + output.prompt_token_id_logprobs, + ref.prompt_token_id_logprobs, + rtol=0.1, + atol=0.2, ) - for output, ids in zip(self_scored, own_ids): - scores = output.prompt_token_id_logprobs + for output in self_scored: + scores = output.prompt_token_id_logprobs[:, 0] for row, target in enumerate(output.prompt_token_ids[1:]): expected = output.prompt_logprobs[row + 1][target].logprob - assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3) + assert scores[row] == pytest.approx(expected, abs=1e-3) def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch): @@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch): """ monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") - prompt = "The capital of France is Paris. " * 20 - sampling_params = SamplingParams( - temperature=0.0, - max_tokens=1, - prompt_logprob_token_ids=[10, 100, 1000], - skip_reading_prefix_cache=False, - ) + text = "The capital of France is Paris. " * 20 with VllmRunner( "Qwen/Qwen3-0.6B", @@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch): enable_prefix_caching=True, gpu_memory_utilization=0.25, ) as vllm_model: + token_ids = vllm_model.llm.get_tokenizer()(text).input_ids + prompt = TokensPrompt(prompt_token_ids=token_ids) + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=1, + prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)), + skip_reading_prefix_cache=False, + ) + first = vllm_model.llm.generate([prompt], sampling_params)[0] assert first.prompt_token_id_logprobs is not None assert math.isfinite(float(first.prompt_token_id_logprobs.min())) diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py index cf6e3a83f7..43f8266ded 100644 --- a/tests/v1/sample/test_trace_replay_params.py +++ b/tests/v1/sample/test_trace_replay_params.py @@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list(): def test_sampling_params_trace_field_rejects_prompt_scoring(): - params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3]) + params = SamplingParams( + trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]] + ) with pytest.raises(ValueError, match="prompt_logprob_token_ids"): params._validate_trace_replay( _make_model_config(vocab_size=100), speculative_config=None diff --git a/vllm/outputs.py b/vllm/outputs.py index 62247d93b2..35f71ec304 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -127,9 +127,9 @@ class RequestOutput: decoder input prompt token ids. prompt_logprobs: The log probabilities to return per prompt token. prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped - [max(prompt_len - 1 - prompt_logprob_start, 0), - len(prompt_logprob_token_ids)]; row i scores them as predictions of - prompt token prompt_logprob_start + i + 1. + [prompt_len - 1 - prompt_logprob_start, num_ids]; row i holds the + logprobs of its own IDs as predictions of prompt token + prompt_logprob_start + i + 1, with -inf in padded columns. outputs: The output sequences of the request. finished: Whether the whole request is finished. metrics: Metrics associated with the request. diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 15657997ab..246e2ce90c 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -11,7 +11,8 @@ from functools import cached_property from typing import Annotated, Any import msgspec -from pydantic import BeforeValidator +import numpy as np +from pydantic import BeforeValidator, GetPydanticSchema, StrictInt from pydantic.dataclasses import dataclass import vllm.envs as envs @@ -20,6 +21,7 @@ from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.utils.mistral import is_mistral_tokenizer +from vllm.utils.torch_utils import make_ndarray_with_pad from vllm.v1.serial_utils import PydanticMsgspecMixin logger = init_logger(__name__) @@ -289,10 +291,14 @@ class SamplingParams( prompt_logprobs: int | None = None """Number of log probabilities to return per prompt token. When set to -1, return all `vocab_size` log probabilities.""" - prompt_logprob_token_ids: list[int] | None = None - """Token IDs to score at each scored causal prompt row, where row i scores - them as predictions of prompt token i + 1. The last prompt row is excluded, - so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows.""" + prompt_logprob_token_ids: ( + Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))] + | None + ) = None + """Token IDs to score per causal prompt row: an integer array of shape + [num_rows, num_ids] or nested lists; -1 entries (padding) score -inf. + Row i scores its IDs as predictions of prompt token + prompt_logprob_start + i + 1, so the last prompt row is excluded.""" prompt_logprob_start: int | None = None """First causal prompt row to score; defaults to the first row.""" logprob_token_ids: list[int] | None = None @@ -404,7 +410,7 @@ class SamplingParams( min_tokens: int = 0, logprobs: int | None = None, prompt_logprobs: int | None = None, - prompt_logprob_token_ids: list[int] | None = None, + prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None, prompt_logprob_start: int | None = None, detokenize: bool = True, skip_special_tokens: bool = True, @@ -916,42 +922,58 @@ class SamplingParams( ) # Validate prompt_logprob_token_ids. - if self.prompt_logprob_token_ids is not None: - n = len(self.prompt_logprob_token_ids) - if n == 0: + ids = self.prompt_logprob_token_ids + if ids is not None: + shape: tuple[int, ...] = () + if isinstance(ids, list) and all(isinstance(row, list) for row in ids): + shape = (len(ids), max(map(len, ids), default=0)) + elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu": + shape = ids.shape + if len(shape) != 2 or 0 in shape: raise VLLMValidationError( - "prompt_logprob_token_ids must not be empty.", + "prompt_logprob_token_ids must be a non-empty integer array " + "of shape [num_rows, num_ids].", parameter="prompt_logprob_token_ids", - value=n, + value=getattr(ids, "shape", type(ids).__name__), ) - if n > max_logprobs: + max_rows = model_config.max_model_len - 1 + if shape[0] > max_rows: raise VLLMValidationError( - f"Requested prompt_logprob_token_ids of length {n}, " - f"which is greater than max allowed: {max_logprobs}. " - f"Set max_logprobs (--max-logprobs) to at least {n}.", + f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt " + f"has at most {max_rows} scored rows (max_model_len - 1).", parameter="prompt_logprob_token_ids", - value=n, + value=shape[0], ) + num_ids = shape[1] + if num_ids > max_logprobs: + raise VLLMValidationError( + f"Requested {num_ids} token ids per row in " + f"prompt_logprob_token_ids, which is greater than max allowed: " + f"{max_logprobs}. " + f"Set max_logprobs (--max-logprobs) to at least {num_ids}.", + parameter="prompt_logprob_token_ids", + value=num_ids, + ) + if isinstance(ids, list): + # Pad only after the checks above bound the allocation. + try: + ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids) + except (TypeError, ValueError, OverflowError) as e: + raise VLLMValidationError( + "prompt_logprob_token_ids must contain integer token ids.", + parameter="prompt_logprob_token_ids", + ) from e vocab_size = model_config.get_vocab_size() - invalid_token_ids = [ - token_id - for token_id in self.prompt_logprob_token_ids - if token_id < 0 or token_id >= vocab_size - ] - if invalid_token_ids: + lo, hi = int(ids.min()), int(ids.max()) + if lo < -1 or hi >= vocab_size: raise VLLMValidationError( - f"token_id(s) {invalid_token_ids} in " - f"prompt_logprob_token_ids contain out-of-vocab token ids. " - f"Vocabulary size: {vocab_size}", + "prompt_logprob_token_ids contain out-of-vocab token ids " + f"(-1 pads a row). Vocabulary size: {vocab_size}", parameter="prompt_logprob_token_ids", - value=invalid_token_ids, - ) - if len(set(self.prompt_logprob_token_ids)) != n: - raise VLLMValidationError( - "prompt_logprob_token_ids must not contain duplicates.", - parameter="prompt_logprob_token_ids", - value=self.prompt_logprob_token_ids, + value=[lo, hi], ) + # In-vocab IDs fit int32, like the runner's other token ID buffers. + self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32) if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0: raise VLLMValidationError( "prompt_logprob_start must be non-negative.", diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index fa68eb8ec2..8b5a69bcf4 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -423,6 +423,18 @@ class InputProcessor: parameter="routed_experts_prompt_start", value=sampling_params.routed_experts_prompt_start, ) + rows = sampling_params.prompt_logprob_token_ids + if rows is not None: + start = sampling_params.prompt_logprob_start or 0 + num_rows = max(prompt_len - 1 - start, 0) + if len(rows) != num_rows: + raise VLLMValidationError( + f"prompt_logprob_token_ids has {len(rows)} rows, but the " + f"prompt has {num_rows} scored rows " + f"(prompt_len - 1 - prompt_logprob_start).", + parameter="prompt_logprob_token_ids", + value=len(rows), + ) # If unset max tokens, then generate up to the max_model_len. if sampling_params.max_tokens is None: sampling_params.max_tokens = ( diff --git a/vllm/v1/serial_utils.py b/vllm/v1/serial_utils.py index 882bc44917..13d725410a 100644 --- a/vllm/v1/serial_utils.py +++ b/vllm/v1/serial_utils.py @@ -532,7 +532,7 @@ class PydanticMsgspecMixin: called on every validation. """ msgspec_fields = {f.name: f for f in msgspec.structs.fields(source_type)} - type_hints = get_type_hints(source_type) + type_hints = get_type_hints(source_type, include_extras=True) # Build the Pydantic typed_dict_field for each msgspec field fields = {} diff --git a/vllm/v1/worker/gpu/sample/prompt_logprob.py b/vllm/v1/worker/gpu/sample/prompt_logprob.py index 24fdb6188e..0fbdbc4134 100644 --- a/vllm/v1/worker/gpu/sample/prompt_logprob.py +++ b/vllm/v1/worker/gpu/sample/prompt_logprob.py @@ -22,7 +22,8 @@ CHUNK_SIZE = 1024 @dataclass class _TokenIdScores: - token_ids: torch.Tensor # [num_ids], on device + token_ids: torch.Tensor # [num_rows, num_ids], on device + pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns start: int # first prompt row to score scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk @@ -52,11 +53,13 @@ class PromptLogprobsWorker: if uses_prompt_logprobs: self.in_progress_prompt_logprobs[req_id] = [] if sampling_params.prompt_logprob_token_ids is not None: + ids = sampling_params.prompt_logprob_token_ids + # Decoded arrays may be read-only; torch.from_numpy warns on those. + if not ids.flags.writeable: + ids = ids.copy() + ids = async_tensor_h2d(ids, self.device, torch.int64) self.token_id_scores[req_id] = _TokenIdScores( - async_tensor_h2d( - sampling_params.prompt_logprob_token_ids, self.device, torch.int64 - ), - sampling_params.prompt_logprob_start or 0, + ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0 ) def remove_request(self, req_id: str) -> None: @@ -85,6 +88,7 @@ class PromptLogprobsWorker: # Skip decode steps and, as compute_prompt_logprobs does, requests # resumed after preemption: their scores were already emitted. if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]: + self.token_id_scores.pop(req_id, None) continue if req.scores is None: if chunk_start > req.start: @@ -93,10 +97,7 @@ class PromptLogprobsWorker: # emitting a buffer with unwritten rows. self.token_id_scores.pop(req_id, None) continue - req.scores = hidden_states.new_empty( - (max(prompt_len - 1 - req.start, 0), len(req.token_ids)), - dtype=torch.float32, - ) + req.scores = torch.empty_like(req.token_ids, dtype=torch.float32) # The last prompt row predicts the first decode token; skip it. lo = max(req.start, chunk_start) hi = min(chunk_end, prompt_len - 1) @@ -104,15 +105,16 @@ class PromptLogprobsWorker: for a in range(lo, hi, CHUNK_SIZE): b = min(a + CHUNK_SIZE, hi) logits = logits_fn(hidden_states[base + a : base + b]) - ids = req.token_ids.expand(b - a, -1) + ids = req.token_ids[a - req.start : b - req.start] req.scores[a - req.start : b - req.start] = ( logits.gather(-1, ids) if logits_mode else compute_token_logprobs(logits, ids) ) if chunk_end >= prompt_len: + req.scores.masked_fill_(req.pad, float("-inf")) out[req_id] = req.scores - req.scores = None + self.token_id_scores.pop(req_id, None) return out def compute_prompt_logprobs(