[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -0,0 +1,64 @@
|
||||
# Prompt Token ID Logprobs (Teacher Scoring)
|
||||
|
||||
In on-policy distillation, the student records its top-K candidate tokens at
|
||||
every response position, and the teacher provides the log probabilities of
|
||||
exactly those candidates as the distillation target. The caller passes the
|
||||
candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill
|
||||
over the prompt plus response and gathers their log probabilities on the GPU at
|
||||
every scored row, starting from `prompt_logprob_start` (typically the prompt
|
||||
length minus one, so only response positions are scored). The result is a
|
||||
`[rows, K]` matrix whose row `i` scores prompt token
|
||||
`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so
|
||||
it lines up position by position with the student's top-K.
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
llm = LLM(model)
|
||||
output = llm.generate(
|
||||
TokensPrompt(prompt_token_ids=prompt_ids + response_ids),
|
||||
SamplingParams(
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=len(prompt_ids) - 1,
|
||||
),
|
||||
)
|
||||
scores = output[0].prompt_token_id_logprobs
|
||||
# scores.shape == (len(response_ids), len(candidate_ids))
|
||||
# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i])
|
||||
```
|
||||
|
||||
The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint
|
||||
(Python and Rust frontends), which returns the matrix as a base64-encoded
|
||||
`.npy` float32 array:
|
||||
|
||||
```python
|
||||
import base64, io
|
||||
import numpy as np
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"])))
|
||||
```
|
||||
|
||||
The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a
|
||||
logits `--logprobs-mode`, it holds logits instead of log probabilities.
|
||||
|
||||
## Requirements
|
||||
|
||||
- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`).
|
||||
- `len(prompt_logprob_token_ids)` at most `--max-logprobs`.
|
||||
- No `--kv-sharing-fast-prefill`.
|
||||
|
||||
The request skips reading the prefix cache (local and KV connector), since
|
||||
cached rows have no logits to score.
|
||||
|
||||
## Limitations
|
||||
|
||||
- One candidate set per request, shared by every scored row; per-position
|
||||
candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)).
|
||||
- Non-streaming only; `stream=true` requests are rejected.
|
||||
- Not exposed through the OpenAI-compatible endpoints.
|
||||
- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False`
|
||||
with a cache hit), the result is `None` rather than a partial matrix.
|
||||
Generated
+1
@@ -5968,6 +5968,7 @@ dependencies = [
|
||||
"asynk-strim-attr",
|
||||
"auto_enums",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"clap",
|
||||
"educe",
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
|
||||
/// the first output of a request that asked for them.
|
||||
#[serde(default)]
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
|
||||
self.new_sampling_mask = (self.new_sampling_mask.take())
|
||||
.map(|value| value.resolve(frames, "new_sampling_mask"))
|
||||
.transpose()?;
|
||||
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
|
||||
value
|
||||
.resolve_aux_frame(frames)
|
||||
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -531,6 +541,7 @@ mod tests {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -116,6 +116,7 @@ class EngineCoreOutput(
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
new_sampling_mask: object | None = None
|
||||
spec_decode_metrics: object | None = None
|
||||
prompt_token_id_logprobs: object | None = None
|
||||
|
||||
|
||||
class ExtendedEngineCoreOutput(EngineCoreOutput):
|
||||
|
||||
@@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
|
||||
|
||||
use crate::error::Result;
|
||||
@@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Vec<u32>,
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub logprobs: Option<Logprobs>,
|
||||
pub finish_reason: FinishReason,
|
||||
@@ -62,6 +64,8 @@ pub struct GeneratePromptInfo {
|
||||
/// Prompt logprobs returned by engine-core for scored prompt positions,
|
||||
/// when requested.
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
/// Log probabilities of `prompt_logprob_token_ids`, when requested.
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
}
|
||||
|
||||
/// The reason a request finished.
|
||||
@@ -206,6 +210,7 @@ impl GenerateOutput {
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: ids,
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -247,6 +252,7 @@ impl GenerateOutputStream {
|
||||
pending_prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
raw_stream,
|
||||
request_metrics,
|
||||
@@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream {
|
||||
info.prompt_logprobs =
|
||||
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
|
||||
}
|
||||
if let Some(info) = &mut self.pending_prompt_info
|
||||
&& info.prompt_token_id_logprobs.is_none()
|
||||
{
|
||||
info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs;
|
||||
}
|
||||
|
||||
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
|
||||
let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap());
|
||||
@@ -361,6 +372,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
pin_mut!(stream);
|
||||
let mut prompt_token_ids = None;
|
||||
let mut prompt_logprobs = None;
|
||||
let mut prompt_token_id_logprobs = None;
|
||||
let mut cached_token_count = 0;
|
||||
let mut collected: Option<CollectedGenerateOutput> = None;
|
||||
|
||||
@@ -374,6 +386,9 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
if prompt_logprobs.is_none() {
|
||||
prompt_logprobs = info.prompt_logprobs;
|
||||
}
|
||||
if prompt_token_id_logprobs.is_none() {
|
||||
prompt_token_id_logprobs = info.prompt_token_id_logprobs;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(existing) = collected.as_mut() {
|
||||
@@ -393,6 +408,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
request_id: output.request_id,
|
||||
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
prompt_token_id_logprobs: prompt_token_id_logprobs.take(),
|
||||
token_ids: output.token_ids,
|
||||
logprobs: output.logprobs,
|
||||
finish_reason: FinishReason::Error,
|
||||
|
||||
@@ -277,6 +277,7 @@ async fn generate_streams_outputs() {
|
||||
Some(GeneratePromptInfo {
|
||||
prompt_token_ids: vec![11, 22].into(),
|
||||
prompt_logprobs: Some(prompt_logprobs()),
|
||||
prompt_token_id_logprobs: None,
|
||||
})
|
||||
);
|
||||
assert_eq!(first.token_ids, vec![1, 2]);
|
||||
@@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids: vec![33, 44],
|
||||
logprobs: None,
|
||||
|
||||
@@ -9,6 +9,7 @@ anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
auto_enums.workspace = true
|
||||
axum.workspace = true
|
||||
base64.workspace = true
|
||||
educe.workspace = true
|
||||
futures.workspace = true
|
||||
http-body.workspace = true
|
||||
|
||||
@@ -16,12 +16,15 @@ use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use base64::Engine as _;
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::dtype::TensorDtype;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
|
||||
};
|
||||
@@ -330,6 +333,11 @@ fn collect_generate(
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
.prompt_token_id_logprobs
|
||||
.as_ref()
|
||||
.map(npy_base64)
|
||||
.transpose()?,
|
||||
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
|
||||
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Encode a float32 matrix as Python `numpy2base64` does.
|
||||
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
|
||||
let (TensorDtype::F32, Some(data), [rows, cols]) = (
|
||||
array.dtype.scalar,
|
||||
array.data.as_raw_view(),
|
||||
array.shape.as_slice(),
|
||||
) else {
|
||||
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
|
||||
};
|
||||
let mut header =
|
||||
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
|
||||
header.push_str(&" ".repeat(21 - rows.to_string().len()));
|
||||
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
|
||||
header.push('\n');
|
||||
|
||||
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
|
||||
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
|
||||
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
|
||||
npy.extend_from_slice(header.as_bytes());
|
||||
npy.extend_from_slice(data);
|
||||
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
|
||||
}
|
||||
|
||||
fn format_token_id(token_id: u32) -> String {
|
||||
format!("token_id:{token_id}")
|
||||
}
|
||||
@@ -509,6 +540,7 @@ mod tests {
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22_u32]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids: vec![33],
|
||||
logprobs: None,
|
||||
@@ -592,6 +624,7 @@ mod tests {
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -747,6 +780,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -781,6 +815,17 @@ mod tests {
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_token_id_logprobs_match_numpy2base64() {
|
||||
let scores =
|
||||
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
|
||||
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
|
||||
assert_eq!(
|
||||
npy_base64(&scores).unwrap(),
|
||||
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_returns_spec_decode_metrics() {
|
||||
let metrics = RequestSpecDecodeMetrics {
|
||||
@@ -792,6 +837,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-metrics".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -879,6 +925,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -914,6 +961,7 @@ mod tests {
|
||||
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
|
||||
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseChoice>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
|
||||
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
|
||||
pub prompt_token_id_logprobs: Option<String>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
|
||||
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
|
||||
}
|
||||
}
|
||||
|
||||
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"`prompt_logprob_token_ids` are not available when `stream=true`."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -169,6 +176,17 @@ mod tests {
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"token_ids": [11, 22],
|
||||
"stream": true,
|
||||
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
logprob_token_ids.as_deref(),
|
||||
prompt_logprob_token_ids.as_deref(),
|
||||
prompt_logprob_start,
|
||||
sampling_limits,
|
||||
)?;
|
||||
validate_repetition_detection(repetition_detection.as_ref())?;
|
||||
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -646,6 +652,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -701,6 +709,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -870,6 +880,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -935,6 +947,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1008,6 +1022,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1122,6 +1138,44 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
|
||||
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
prompt_logprob_token_ids: ids,
|
||||
prompt_logprob_start: start,
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
};
|
||||
let rejects = |ids, start| match lower(ids, start) {
|
||||
Err(error) => error.to_string(),
|
||||
Ok(_) => panic!("expected rejection"),
|
||||
};
|
||||
|
||||
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
|
||||
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
|
||||
assert_eq!(params.prompt_logprob_start, Some(1));
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
|
||||
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&[
|
||||
rejects(Some(vec![]), None),
|
||||
rejects(Some((0..21).collect()), None),
|
||||
rejects(Some(vec![1000]), None),
|
||||
rejects(Some(vec![1, 1]), None),
|
||||
rejects(None, Some(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
@@ -1259,6 +1313,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
@@ -35,6 +36,12 @@ pub enum LogprobsError {
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
#[error("prompt_logprob_token_ids must not be empty.")]
|
||||
EmptyPromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_token_ids must not contain duplicates.")]
|
||||
DuplicatePromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
|
||||
PromptLogprobStartWithoutTokenIds,
|
||||
}
|
||||
|
||||
/// Validate logprobs count sampling parameters.
|
||||
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_start: Option<u32>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let vocab_size = sampling_limits.model_vocab_size;
|
||||
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
|
||||
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprobs_count(
|
||||
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
|
||||
max_logprobs,
|
||||
vocab_size,
|
||||
"prompt_logprob_token_ids",
|
||||
)?;
|
||||
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids)
|
||||
}
|
||||
|
||||
fn validate_prompt_logprob_token_ids(
|
||||
token_ids: Option<&[u32]>,
|
||||
start: Option<u32>,
|
||||
) -> Result<(), LogprobsError> {
|
||||
match token_ids {
|
||||
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
|
||||
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
|
||||
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
|
||||
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"prompt_logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.model_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"bad_words",
|
||||
|
||||
@@ -97,6 +97,11 @@ pub struct SamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
/// explicit user override.
|
||||
pub min_p: Option<f32>,
|
||||
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: None,
|
||||
frequency_penalty: None,
|
||||
presence_penalty: None,
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": sampling_params,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
|
||||
@@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logprobs import create_sample_logprobs
|
||||
from vllm.v1.engine import EngineCoreOutput
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.outputs import LogprobsLists
|
||||
|
||||
@@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row():
|
||||
|
||||
pos = processor.logprobs[0]
|
||||
assert set(pos.keys()) == {7, 11, 13}
|
||||
|
||||
|
||||
def test_prompt_token_id_logprobs_are_popped_once():
|
||||
"""DELTA outputs carry fixed-ID prompt scores exactly once."""
|
||||
processor = _make_processor(num_logprobs=1)
|
||||
assert processor.pop_prompt_token_id_logprobs() is None
|
||||
|
||||
processor.update_from_output(
|
||||
EngineCoreOutput(
|
||||
request_id="req-0",
|
||||
new_token_ids=[],
|
||||
prompt_token_id_logprobs=torch.tensor(
|
||||
[[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
scores = processor.pop_prompt_token_id_logprobs()
|
||||
assert scores is not None
|
||||
assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]]
|
||||
assert processor.pop_prompt_token_id_logprobs() is None
|
||||
|
||||
@@ -6,7 +6,9 @@ import math
|
||||
from collections.abc import Generator
|
||||
from types import SimpleNamespace
|
||||
from typing import get_args
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
|
||||
@@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
|
||||
yield hf_model
|
||||
|
||||
|
||||
def _model_config(vocab_size: int = 10):
|
||||
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
|
||||
return SimpleNamespace(
|
||||
max_logprobs=20,
|
||||
max_logprobs=max_logprobs,
|
||||
logits_processors=None,
|
||||
is_diffusion=False,
|
||||
get_vocab_size=lambda: vocab_size,
|
||||
@@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
|
||||
"""The candidate count is bounded by max_logprobs."""
|
||||
model_config = _model_config(vocab_size=100, max_logprobs=4)
|
||||
|
||||
def verify(**kwargs):
|
||||
SamplingParams(**kwargs).verify(
|
||||
model_config,
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
|
||||
|
||||
# The error names the value to raise max_logprobs to.
|
||||
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
|
||||
|
||||
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
|
||||
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
|
||||
verify(prompt_logprob_start=3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
"""Only the V2 runner scores them; the V1 runner would return None, and
|
||||
kv-sharing fast prefill would score rows the cross-decoder never ran."""
|
||||
processor = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False
|
||||
),
|
||||
vllm_config=SimpleNamespace(
|
||||
reasoning_config=None,
|
||||
use_v2_model_runner=False,
|
||||
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
|
||||
),
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
processor.vllm_config.use_v2_model_runner = True
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
|
||||
with pytest.raises(VLLMValidationError, match="fast-prefill"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
|
||||
|
||||
def test_none_logprobs(vllm_model, example_prompts):
|
||||
"""Engine should return `logprobs` and `prompt_logprobs` as `None`.
|
||||
|
||||
@@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
print(f"Test passed with {preemptions} preemptions")
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompts = [
|
||||
"The following numbers of the sequence "
|
||||
+ ", ".join(str(i) for i in range(10))
|
||||
+ " are:",
|
||||
"In one word, the capital of France is ",
|
||||
] + [f"Tell me about the number {i}: " for i in range(32)]
|
||||
|
||||
start = 2
|
||||
candidate_ids = [10, 100, 1000, 10000]
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
enable_chunked_prefill=True,
|
||||
max_num_batched_tokens=48, # Force prefill chunking
|
||||
num_gpu_blocks_override=33, # Force preemptions (32 usable + null block)
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
metrics_before = vllm_model.llm.get_metrics()
|
||||
outputs = vllm_model.llm.generate(prompts, sampling_params)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
assert scores is not None, f"Output {i} missing fixed-ID scores"
|
||||
expected_shape = (
|
||||
len(output.prompt_token_ids) - 1 - start,
|
||||
len(candidate_ids),
|
||||
)
|
||||
assert scores.shape == expected_shape, (
|
||||
f"Output {i} scored {scores.shape}, expected {expected_shape}"
|
||||
)
|
||||
assert math.isfinite(float(scores.min()))
|
||||
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
|
||||
|
||||
metrics_after = vllm_model.llm.get_metrics()
|
||||
preemptions_before = next(
|
||||
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
|
||||
)
|
||||
preemptions_after = next(
|
||||
(m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0
|
||||
)
|
||||
assert preemptions_after - preemptions_before > 0, (
|
||||
"Test did not trigger any preemptions"
|
||||
)
|
||||
|
||||
# Row alignment is numerical: chunked and preempted scores must match an
|
||||
# unchunked, unpreempted run of the same requests. Batch composition moves
|
||||
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_logprobs=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as reference:
|
||||
reference_outputs = reference.llm.generate(prompts, sampling_params)
|
||||
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
|
||||
self_scored = reference.llm.generate(
|
||||
prompts,
|
||||
[
|
||||
SamplingParams(
|
||||
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
|
||||
)
|
||||
for ids in own_ids
|
||||
],
|
||||
)
|
||||
for output, ref in zip(outputs, reference_outputs):
|
||||
np.testing.assert_allclose(
|
||||
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
|
||||
)
|
||||
for output, ids in zip(self_scored, own_ids):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
for row, target in enumerate(output.prompt_token_ids[1:]):
|
||||
expected = output.prompt_logprobs[row + 1][target].logprob
|
||||
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
"""A prefill that starts past the first scored row returns no scores.
|
||||
|
||||
Such a prefill (here a prefix-cache hit the caller opted back into) never
|
||||
computes the leading rows, so emitting the buffer would return values the
|
||||
model never produced for them.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompt = "The capital of France is Paris. " * 20
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=[10, 100, 1000],
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
first = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert first.prompt_token_id_logprobs is not None
|
||||
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
|
||||
|
||||
# The re-send is served from the prefix cache, so its prefill starts
|
||||
# past row 0 and the leading rows are never computed.
|
||||
second = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert second.prompt_token_id_logprobs is None
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_token_logprobs_large_batch_int64_row_offset():
|
||||
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
|
||||
|
||||
@@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list():
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_prompt_scoring():
|
||||
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
|
||||
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_requires_single_output():
|
||||
params = SamplingParams(n=2, trace_decode_token_ids=[1])
|
||||
with pytest.raises(ValueError, match="requires n=1"):
|
||||
|
||||
@@ -381,6 +381,7 @@ class GenerateResponse(BaseModel):
|
||||
choices: list[GenerateResponseChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_id_logprobs: str | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -138,6 +138,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
f"sampling_params.n must be at most the server's max_num_seqs "
|
||||
f"({max_num_seqs}), got {sampling_params.n}."
|
||||
)
|
||||
# The stream schema has no field for the scores.
|
||||
if request.stream and sampling_params.prompt_logprob_token_ids is not None:
|
||||
return self.create_error_response(
|
||||
"prompt_logprob_token_ids are not available when stream=true."
|
||||
)
|
||||
if self.force_no_detokenize and sampling_params.stop:
|
||||
# SamplingParams rejects stop with detokenize=False at request
|
||||
# validation, but this server forces detokenize=False afterwards,
|
||||
@@ -389,6 +394,11 @@ class ServingTokens(GenerateBaseServing):
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
prompt_token_id_logprobs=(
|
||||
numpy2base64(final_res.prompt_token_id_logprobs)
|
||||
if final_res.prompt_token_id_logprobs is not None
|
||||
else None
|
||||
),
|
||||
prompt_token_ids=(
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
|
||||
@@ -117,6 +117,10 @@ class RequestOutput:
|
||||
For encoder/decoder models, this is the
|
||||
decoder input prompt token ids.
|
||||
prompt_logprobs: The log probabilities to return per prompt token.
|
||||
prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped
|
||||
[max(prompt_len - 1 - prompt_logprob_start, 0),
|
||||
len(prompt_logprob_token_ids)]; row i scores them as predictions of
|
||||
prompt token prompt_logprob_start + i + 1.
|
||||
outputs: The output sequences of the request.
|
||||
finished: Whether the whole request is finished.
|
||||
metrics: Metrics associated with the request.
|
||||
@@ -148,6 +152,7 @@ class RequestOutput:
|
||||
num_cached_tokens: int | None = None,
|
||||
num_cache_creation_tokens: int | None = None,
|
||||
*,
|
||||
prompt_token_id_logprobs: np.ndarray | None = None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
# Forward compatibility, code that uses args added in new release can
|
||||
@@ -162,6 +167,7 @@ class RequestOutput:
|
||||
self.prompt = prompt
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.prompt_logprobs = prompt_logprobs
|
||||
self.prompt_token_id_logprobs = prompt_token_id_logprobs
|
||||
self.outputs = outputs
|
||||
self.finished = finished
|
||||
self.metrics = metrics
|
||||
@@ -215,6 +221,7 @@ class RequestOutput:
|
||||
f"encoder_prompt={self.encoder_prompt!r}, "
|
||||
f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, "
|
||||
f"prompt_logprobs={self.prompt_logprobs}, "
|
||||
f"prompt_token_id_logprobs={self.prompt_token_id_logprobs}, "
|
||||
f"outputs={self.outputs}, "
|
||||
f"finished={self.finished}, "
|
||||
f"metrics={self.metrics}, "
|
||||
|
||||
+68
-1
@@ -289,6 +289,12 @@ class SamplingParams(
|
||||
prompt_logprobs: int | None = None
|
||||
"""Number of log probabilities to return per prompt token.
|
||||
When set to -1, return all `vocab_size` log probabilities."""
|
||||
prompt_logprob_token_ids: list[int] | None = None
|
||||
"""Token IDs to score at each scored causal prompt row, where row i scores
|
||||
them as predictions of prompt token i + 1. The last prompt row is excluded,
|
||||
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
|
||||
prompt_logprob_start: int | None = None
|
||||
"""First causal prompt row to score; defaults to the first row."""
|
||||
logprob_token_ids: list[int] | None = None
|
||||
"""Specific token IDs to return logprobs for. More efficient than
|
||||
logprobs=-1 when you only need logprobs for a small set of tokens.
|
||||
@@ -398,6 +404,8 @@ class SamplingParams(
|
||||
min_tokens: int = 0,
|
||||
logprobs: int | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
prompt_logprob_token_ids: list[int] | None = None,
|
||||
prompt_logprob_start: int | None = None,
|
||||
detokenize: bool = True,
|
||||
skip_special_tokens: bool = True,
|
||||
spaces_between_special_tokens: bool = True,
|
||||
@@ -464,6 +472,8 @@ class SamplingParams(
|
||||
min_tokens=min_tokens,
|
||||
logprobs=logprobs,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
prompt_logprob_token_ids=prompt_logprob_token_ids,
|
||||
prompt_logprob_start=prompt_logprob_start,
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
detokenize=detokenize,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
@@ -550,7 +560,10 @@ class SamplingParams(
|
||||
# If prefix caching is enabled,
|
||||
# the output of prompt logprobs may less than n_prompt_tokens,
|
||||
# we need to skip reading cache at this request.
|
||||
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
|
||||
self.skip_reading_prefix_cache = (
|
||||
self.prompt_logprobs is not None
|
||||
or self.prompt_logprob_token_ids is not None
|
||||
)
|
||||
|
||||
def _verify_args(self) -> None:
|
||||
_verify_num_sequences(self.n, "n")
|
||||
@@ -902,6 +915,56 @@ class SamplingParams(
|
||||
value=num_prompt_logprobs,
|
||||
)
|
||||
|
||||
# Validate prompt_logprob_token_ids.
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
n = len(self.prompt_logprob_token_ids)
|
||||
if n == 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
)
|
||||
if n > max_logprobs:
|
||||
raise VLLMValidationError(
|
||||
f"Requested prompt_logprob_token_ids of length {n}, "
|
||||
f"which is greater than max allowed: {max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {n}.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
)
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
for token_id in self.prompt_logprob_token_ids
|
||||
if token_id < 0 or token_id >= vocab_size
|
||||
]
|
||||
if invalid_token_ids:
|
||||
raise VLLMValidationError(
|
||||
f"token_id(s) {invalid_token_ids} in "
|
||||
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
|
||||
f"Vocabulary size: {vocab_size}",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
if len(set(self.prompt_logprob_token_ids)) != n:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=self.prompt_logprob_token_ids,
|
||||
)
|
||||
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start must be non-negative.",
|
||||
parameter="prompt_logprob_start",
|
||||
value=self.prompt_logprob_start,
|
||||
)
|
||||
elif self.prompt_logprob_start is not None:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
parameter="prompt_logprob_start",
|
||||
value=self.prompt_logprob_start,
|
||||
)
|
||||
|
||||
def _validate_stop_token_ids(self, model_config: ModelConfig) -> None:
|
||||
"""Validate stop_token_ids are within vocabulary range."""
|
||||
if not self.stop_token_ids:
|
||||
@@ -969,6 +1032,10 @@ class SamplingParams(
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with prompt_logprobs."
|
||||
)
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with prompt_logprob_token_ids."
|
||||
)
|
||||
if speculative_config is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with speculative decoding."
|
||||
|
||||
@@ -1972,6 +1972,9 @@ class Scheduler(SchedulerInterface):
|
||||
sampled_token_ids = model_runner_output.sampled_token_ids
|
||||
logprobs = model_runner_output.logprobs
|
||||
prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
|
||||
prompt_token_id_logprobs_dict = (
|
||||
model_runner_output.prompt_token_id_logprobs_dict
|
||||
)
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
|
||||
pooler_outputs = model_runner_output.pooler_output
|
||||
num_nans_in_logits = model_runner_output.num_nans_in_logits
|
||||
@@ -2194,6 +2197,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Get prompt logprobs for this request.
|
||||
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
|
||||
prompt_token_id_logprobs = prompt_token_id_logprobs_dict.get(req_id)
|
||||
if should_emit_output:
|
||||
# Add EngineCoreOutput for this Request.
|
||||
outputs[request.client_index].append(
|
||||
@@ -2204,6 +2208,7 @@ class Scheduler(SchedulerInterface):
|
||||
new_logprobs=new_logprobs,
|
||||
new_sampling_mask=new_sampling_mask,
|
||||
new_prompt_logprobs_tensors=prompt_logprobs_tensors,
|
||||
prompt_token_id_logprobs=prompt_token_id_logprobs,
|
||||
pooling_output=pooler_output,
|
||||
stop_reason=request.stop_reason,
|
||||
events=request.take_events(),
|
||||
@@ -2223,6 +2228,7 @@ class Scheduler(SchedulerInterface):
|
||||
else:
|
||||
# Invariant: EngineCore returns no partial prefill outputs.
|
||||
assert not prompt_logprobs_tensors
|
||||
assert prompt_token_id_logprobs is None
|
||||
|
||||
# Remove the stopped requests from the running and waiting queues.
|
||||
if stopped_running_reqs:
|
||||
|
||||
@@ -236,6 +236,9 @@ class EngineCoreOutput(
|
||||
# Appended last so `array_like` positional serialization stays compatible.
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
|
||||
|
||||
# [num_scored_rows, num_token_ids]; appended last for array_like compatibility.
|
||||
prompt_token_id_logprobs: torch.Tensor | None = None
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.finish_reason is not None
|
||||
|
||||
@@ -131,6 +131,19 @@ class InputProcessor:
|
||||
self.structured_outputs_config,
|
||||
self.tokenizer,
|
||||
)
|
||||
if params.prompt_logprob_token_ids is not None:
|
||||
if not self.vllm_config.use_v2_model_runner:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids requires the V2 model runner "
|
||||
"(VLLM_USE_V2_MODEL_RUNNER=1).",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
)
|
||||
if self.vllm_config.cache_config.kv_sharing_fast_prefill:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids is incorrect with "
|
||||
"--kv-sharing-fast-prefill; disable it for scoring.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
)
|
||||
|
||||
self.validate_logits_processors_params(params)
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import itertools
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import (
|
||||
FlatLogprobs,
|
||||
@@ -38,6 +40,8 @@ class LogprobsProcessor:
|
||||
cumulative_logprob: float | None
|
||||
num_logprobs: int | None
|
||||
num_prompt_logprobs: int | None
|
||||
# [num_scored_rows, num_token_ids], set once on the final prefill chunk.
|
||||
prompt_token_id_logprobs: np.ndarray | None = None
|
||||
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
@@ -204,6 +208,12 @@ class LogprobsProcessor:
|
||||
self.prompt_logprobs = []
|
||||
return plp
|
||||
|
||||
def pop_prompt_token_id_logprobs(self) -> np.ndarray | None:
|
||||
"""Pop and return the fixed-ID prompt scores."""
|
||||
scores = self.prompt_token_id_logprobs
|
||||
self.prompt_token_id_logprobs = None
|
||||
return scores
|
||||
|
||||
@staticmethod
|
||||
def _get_sampled_context_ids(
|
||||
logprobs_source: SampleLogprobs | PromptLogprobs | None,
|
||||
@@ -352,3 +362,5 @@ class LogprobsProcessor:
|
||||
self._update_sample_logprobs(output.new_logprobs)
|
||||
if output.new_prompt_logprobs_tensors is not None:
|
||||
self._update_prompt_logprobs(output.new_prompt_logprobs_tensors)
|
||||
if output.prompt_token_id_logprobs is not None:
|
||||
self.prompt_token_id_logprobs = output.prompt_token_id_logprobs.numpy()
|
||||
|
||||
@@ -390,8 +390,12 @@ class RequestState:
|
||||
if self.output_kind == RequestOutputKind.DELTA:
|
||||
# Side effect: logprobs processor forgets prompt logprobs
|
||||
prompt_logprobs = self.logprobs_processor.pop_prompt_logprobs()
|
||||
prompt_token_id_logprobs = (
|
||||
self.logprobs_processor.pop_prompt_token_id_logprobs()
|
||||
)
|
||||
else:
|
||||
prompt_logprobs = self.logprobs_processor.prompt_logprobs
|
||||
prompt_token_id_logprobs = self.logprobs_processor.prompt_token_id_logprobs
|
||||
|
||||
return RequestOutput(
|
||||
request_id=external_req_id, # request_id is what was provided externally
|
||||
@@ -399,6 +403,7 @@ class RequestState:
|
||||
prompt=self.prompt,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
prompt_token_id_logprobs=prompt_token_id_logprobs,
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
|
||||
@@ -283,6 +283,9 @@ class ModelRunnerOutput:
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
# req_id -> [num_scored_rows, num_token_ids] prompt_logprob_token_ids scores.
|
||||
prompt_token_id_logprobs_dict: dict[str, torch.Tensor] = field(default_factory=dict)
|
||||
|
||||
# [num_reqs, hidden_size]
|
||||
pooler_output: list[torch.Tensor | None] | None = None
|
||||
|
||||
|
||||
@@ -156,6 +156,10 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
k: v.to_cpu_nonblocking() if v is not None else None
|
||||
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
|
||||
}
|
||||
token_id_logprobs = self.model_runner_output.prompt_token_id_logprobs_dict
|
||||
self.prompt_token_id_logprobs_dict = {
|
||||
k: v.to("cpu", non_blocking=True) for k, v in token_id_logprobs.items()
|
||||
}
|
||||
if self.pending_aux_output is not None:
|
||||
self.pending_aux_output.enqueue_cpu_copy(
|
||||
num_sampled=self.num_sampled_tokens_np,
|
||||
@@ -194,6 +198,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
if self.logprobs_tensors is not None:
|
||||
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
|
||||
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
|
||||
self.model_runner_output.prompt_token_id_logprobs_dict = (
|
||||
self.prompt_token_id_logprobs_dict
|
||||
)
|
||||
if self.pending_aux_output is not None:
|
||||
self.model_runner_output.aux_output_connector_output = (
|
||||
self.pending_aux_output.process_output()
|
||||
|
||||
@@ -494,7 +494,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
),
|
||||
)
|
||||
self.prompt_logprobs_worker = PromptLogprobsWorker(
|
||||
self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode
|
||||
self.max_num_reqs,
|
||||
self.device,
|
||||
logprobs_mode=self.model_config.logprobs_mode,
|
||||
)
|
||||
self.structured_outputs_worker = StructuredOutputsWorker(
|
||||
max_num_logits=self.max_num_reqs * self.decode_query_len,
|
||||
@@ -2063,6 +2065,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.prompt_len.np,
|
||||
)
|
||||
prompt_token_id_logprobs_dict = (
|
||||
self.prompt_logprobs_worker.compute_prompt_token_id_logprobs(
|
||||
self.model.compute_logits,
|
||||
hidden_states,
|
||||
input_batch,
|
||||
self.req_states.prompt_len.np,
|
||||
)
|
||||
)
|
||||
|
||||
# Prepare the model runner output.
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
@@ -2072,6 +2082,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
||||
sampled_token_ids=None, # type: ignore
|
||||
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
|
||||
prompt_token_id_logprobs_dict=prompt_token_id_logprobs_dict,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
)
|
||||
pending_aux_output = None
|
||||
|
||||
@@ -19,6 +19,7 @@ def _topk_log_softmax_kernel(
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
topk_ids_ptr,
|
||||
topk_ids_stride,
|
||||
topk,
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
@@ -49,7 +50,7 @@ def _topk_log_softmax_kernel(
|
||||
k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE)
|
||||
k_mask = k_offset < topk
|
||||
topk_ids = tl.load(
|
||||
topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0
|
||||
topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0
|
||||
)
|
||||
logits = tl.load(row_ptr + topk_ids, mask=k_mask)
|
||||
logits = logits.to(tl.float32)
|
||||
@@ -98,6 +99,7 @@ def compute_token_logprobs(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
num_logprobs,
|
||||
vocab_size,
|
||||
BLOCK_SIZE=1024, # type: ignore
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -8,20 +9,41 @@ import torch
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
from vllm.v1.worker.gpu.sample.logprob import (
|
||||
compute_token_logprobs,
|
||||
compute_topk_scores,
|
||||
)
|
||||
|
||||
CHUNK_SIZE = 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TokenIdScores:
|
||||
token_ids: torch.Tensor # [num_ids], on device
|
||||
start: int # first prompt row to score
|
||||
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
|
||||
|
||||
|
||||
class PromptLogprobsWorker:
|
||||
def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"):
|
||||
def __init__(
|
||||
self,
|
||||
max_num_reqs: int,
|
||||
device: torch.device,
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.device = device
|
||||
self.logprobs_mode = logprobs_mode
|
||||
|
||||
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
|
||||
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
# req_idx -> list of in-progress LogprobsTensors
|
||||
self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {}
|
||||
# req_id -> fixed-ID scoring state
|
||||
self.token_id_scores: dict[str, _TokenIdScores] = {}
|
||||
|
||||
def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams):
|
||||
uses_prompt_logprobs = sampling_params.prompt_logprobs is not None
|
||||
@@ -29,9 +51,69 @@ class PromptLogprobsWorker:
|
||||
self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0
|
||||
if uses_prompt_logprobs:
|
||||
self.in_progress_prompt_logprobs[req_id] = []
|
||||
if sampling_params.prompt_logprob_token_ids is not None:
|
||||
self.token_id_scores[req_id] = _TokenIdScores(
|
||||
async_tensor_h2d(
|
||||
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
|
||||
),
|
||||
sampling_params.prompt_logprob_start or 0,
|
||||
)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
self.in_progress_prompt_logprobs.pop(req_id, None)
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
|
||||
def compute_prompt_token_id_logprobs(
|
||||
self,
|
||||
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
||||
hidden_states: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
prompt_lens: np.ndarray,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Fill scores row (prompt row - req.start) per chunk; emit on the last."""
|
||||
if not self.token_id_scores:
|
||||
return {}
|
||||
logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits")
|
||||
out: dict[str, torch.Tensor] = {}
|
||||
for i, req_id in enumerate(input_batch.req_ids):
|
||||
req = self.token_id_scores.get(req_id)
|
||||
if req is None:
|
||||
continue
|
||||
prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]])
|
||||
chunk_start = int(input_batch.num_computed_prefill_tokens_np[i])
|
||||
chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i])
|
||||
# Skip decode steps and, as compute_prompt_logprobs does, requests
|
||||
# resumed after preemption: their scores were already emitted.
|
||||
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
|
||||
continue
|
||||
if req.scores is None:
|
||||
if chunk_start > req.start:
|
||||
# This prefill starts past the first scored row, so those
|
||||
# rows will never be written; drop the request instead of
|
||||
# emitting a buffer with unwritten rows.
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
req.scores = hidden_states.new_empty(
|
||||
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
# The last prompt row predicts the first decode token; skip it.
|
||||
lo = max(req.start, chunk_start)
|
||||
hi = min(chunk_end, prompt_len - 1)
|
||||
base = int(input_batch.query_start_loc_np[i]) - chunk_start
|
||||
for a in range(lo, hi, CHUNK_SIZE):
|
||||
b = min(a + CHUNK_SIZE, hi)
|
||||
logits = logits_fn(hidden_states[base + a : base + b])
|
||||
ids = req.token_ids.expand(b - a, -1)
|
||||
req.scores[a - req.start : b - req.start] = (
|
||||
logits.gather(-1, ids)
|
||||
if logits_mode
|
||||
else compute_token_logprobs(logits, ids)
|
||||
)
|
||||
if chunk_end >= prompt_len:
|
||||
out[req_id] = req.scores
|
||||
req.scores = None
|
||||
return out
|
||||
|
||||
def compute_prompt_logprobs(
|
||||
self,
|
||||
@@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Since materializing the full prompt logits can take too much memory,
|
||||
# we compute it in chunks.
|
||||
CHUNK_SIZE = 1024
|
||||
token_ids = []
|
||||
scores = []
|
||||
ranks = []
|
||||
|
||||
Reference in New Issue
Block a user