[Frontend] Integer token IDs for generate output logprobs (GenerateLogProbs) (#58181)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@ mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::convert::Infallible;
|
||||
use std::result::Result;
|
||||
use std::sync::Arc;
|
||||
@@ -31,16 +31,17 @@ use vllm_llm::{
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse, MultiModalPlaceholders, PerRequestMetrics, PlaceholderRangeInfo,
|
||||
SpeculativeDecodingMetrics, StreamingSpeculativeDecodingMetrics,
|
||||
GenerateLogProb, GenerateLogProbs, GenerateLogProbsContent, GenerateLogprob, GenerateResponse,
|
||||
GenerateResponseChoice, GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
MultiModalPlaceholders, PerRequestMetrics, PlaceholderRangeInfo, SpeculativeDecodingMetrics,
|
||||
StreamingSpeculativeDecodingMetrics,
|
||||
};
|
||||
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
|
||||
use crate::routes::openai::utils::types::Usage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{resolve_request_context, sse_response};
|
||||
@@ -157,7 +158,7 @@ async fn generate_chunk_stream(
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
logprobs: output_logprobs,
|
||||
// Ignored: raw generate streaming has no prompt-logprobs wire shape.
|
||||
include_prompt_logprobs: _,
|
||||
return_token_ids,
|
||||
@@ -212,13 +213,15 @@ async fn generate_chunk_stream(
|
||||
continue;
|
||||
}
|
||||
|
||||
let logprobs = if include_logprobs && !token_ids.is_empty() {
|
||||
let logprobs = if let Some(requested) = output_logprobs
|
||||
&& !token_ids.is_empty()
|
||||
{
|
||||
let logprobs = output.logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"raw generate stream requested logprobs but generation returned none"
|
||||
)
|
||||
})?;
|
||||
Some(raw_logprobs_to_openai_chat(logprobs)?)
|
||||
Some(raw_logprobs_to_generate(logprobs, requested)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -282,19 +285,19 @@ fn collect_generate(
|
||||
include_usage: _,
|
||||
// Ignored: continuous usage is a streaming-only option.
|
||||
include_continuous_usage: _,
|
||||
include_logprobs,
|
||||
logprobs: output_logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids,
|
||||
}: ResponseOptions,
|
||||
mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
) -> Result<GenerateResponse, ApiError> {
|
||||
let logprobs = if include_logprobs {
|
||||
let logprobs = if let Some(requested) = output_logprobs {
|
||||
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate response requested logprobs but generation returned none".to_string(),
|
||||
)
|
||||
})?;
|
||||
Some(raw_logprobs_to_openai_chat(logprobs)?)
|
||||
Some(raw_logprobs_to_generate(logprobs, requested)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -371,14 +374,17 @@ fn extract_mm_placeholders(features: Option<&[MmFeatureSpec]>) -> Option<MultiMo
|
||||
Some(placeholders)
|
||||
}
|
||||
|
||||
fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result<ChatLogProbs, ApiError> {
|
||||
fn raw_logprobs_to_generate(
|
||||
logprobs: &Logprobs,
|
||||
requested: i32,
|
||||
) -> Result<GenerateLogProbs, ApiError> {
|
||||
let content = logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(position_to_chat_logprobs_content)
|
||||
.map(|position| position_to_generate_logprobs_content(position, requested))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(ChatLogProbs {
|
||||
Ok(GenerateLogProbs {
|
||||
content: Some(content),
|
||||
})
|
||||
}
|
||||
@@ -396,35 +402,47 @@ fn raw_prompt_logprobs_to_maps(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn position_to_chat_logprobs_content(
|
||||
/// `top_logprobs` matches the Python generate server: the engine's entries (the
|
||||
/// sampled token first, then ranks 1..k) with repeated token ids dropped, cut to
|
||||
/// `max(requested, 1)` entries, or all of them for `-1`.
|
||||
fn position_to_generate_logprobs_content(
|
||||
position: &PositionLogprobs,
|
||||
) -> Result<ChatLogProbsContent, ApiError> {
|
||||
requested: i32,
|
||||
) -> Result<GenerateLogProbsContent, ApiError> {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate logprobs position unexpectedly had no token candidates".to_string(),
|
||||
)
|
||||
})?;
|
||||
let token = format_token_id(chosen.token_id);
|
||||
|
||||
Ok(ChatLogProbsContent {
|
||||
token: token.clone(),
|
||||
// One pass: the first entry of each token id wins, as in Python's dict. The
|
||||
// sampled token is the only one the engine can list twice.
|
||||
let mut seen = HashSet::with_capacity(position.entries.len());
|
||||
Ok(GenerateLogProbsContent {
|
||||
token_id: chosen.token_id,
|
||||
logprob: clamp_logprob(chosen.logprob),
|
||||
bytes: Some(token.as_bytes().to_vec()),
|
||||
rank: wire_rank(chosen.rank),
|
||||
top_logprobs: position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let token = format_token_id(entry.token_id);
|
||||
TopLogProb {
|
||||
token: token.clone(),
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
bytes: Some(token.into_bytes()),
|
||||
}
|
||||
.filter(|entry| seen.insert(entry.token_id))
|
||||
.take(usize::try_from(requested).map_or(usize::MAX, |n| n.max(1)))
|
||||
.map(|entry| GenerateLogProb {
|
||||
token_id: entry.token_id,
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
rank: wire_rank(entry.rank),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The engine reports rank 0 for a sampled token whose logprob is NaN (no value
|
||||
/// compares >= NaN). That is not a rank, so it goes on the wire as `None`, the
|
||||
/// way `clamp_logprob` handles the logprob itself.
|
||||
fn wire_rank(rank: u32) -> Option<u32> {
|
||||
(rank > 0).then_some(rank)
|
||||
}
|
||||
|
||||
fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, GenerateLogprob> {
|
||||
position
|
||||
.entries
|
||||
@@ -523,6 +541,80 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
fn position(entries: &[(u32, f32, u32)]) -> PositionLogprobs {
|
||||
PositionLogprobs {
|
||||
entries: entries
|
||||
.iter()
|
||||
.map(|&(token_id, logprob, rank)| {
|
||||
vllm_engine_core_client::protocol::logprobs::TokenLogprob {
|
||||
token_id,
|
||||
logprob,
|
||||
rank,
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn top(position: &PositionLogprobs, requested: i32) -> Vec<(u32, Option<u32>)> {
|
||||
position_to_generate_logprobs_content(position, requested)
|
||||
.unwrap()
|
||||
.top_logprobs
|
||||
.iter()
|
||||
.map(|entry| (entry.token_id, entry.rank))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_top_logprobs_drop_the_repeated_sampled_token() {
|
||||
// Sampled token 7 is also top-1: the engine row is [7, 7, 8].
|
||||
let pos = position(&[(7, -0.1, 1), (7, -0.1, 1), (8, -2.0, 2)]);
|
||||
assert_eq!(top(&pos, 2), vec![(7, Some(1)), (8, Some(2))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_top_logprobs_keep_sampled_first_when_outside_top_k() {
|
||||
// Sampled token 50 has vocab rank 5; it takes a slot and rank 2 drops
|
||||
// out, as on the Python generate server and the OpenAI endpoints.
|
||||
let pos = position(&[(50, -3.0, 5), (10, -0.2, 1), (20, -1.0, 2)]);
|
||||
assert_eq!(top(&pos, 2), vec![(50, Some(5)), (10, Some(1))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_top_logprobs_respect_zero_and_all() {
|
||||
let pos = position(&[(7, -0.1, 1), (7, -0.1, 1), (8, -2.0, 2), (9, -3.0, 3)]);
|
||||
assert_eq!(top(&pos, 0), vec![(7, Some(1))]);
|
||||
assert_eq!(
|
||||
top(&pos, -1),
|
||||
vec![(7, Some(1)), (8, Some(2)), (9, Some(3))]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_top_logprobs_full_vocab_is_linear() {
|
||||
// logprobs = -1 returns the whole vocabulary; deduping it must not be
|
||||
// quadratic in the number of entries.
|
||||
const VOCAB: u32 = 200_000;
|
||||
let mut entries = vec![(5, -0.5, 1)];
|
||||
entries.extend((0..VOCAB).map(|t| (t, -1.0 - t as f32, t + 1)));
|
||||
let pos = position(&entries);
|
||||
let start = std::time::Instant::now();
|
||||
let got = top(&pos, -1);
|
||||
assert!(start.elapsed() < std::time::Duration::from_secs(5));
|
||||
assert_eq!(got.len(), VOCAB as usize);
|
||||
assert_eq!(got[0], (5, Some(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_rank_zero_from_a_nan_logprob_is_none() {
|
||||
// The engine reports rank 0 for a sampled token whose logprob is NaN.
|
||||
let pos = position(&[(7, f32::NAN, 0), (8, -0.2, 1)]);
|
||||
let content = position_to_generate_logprobs_content(&pos, 1).unwrap();
|
||||
assert_eq!(content.rank, None);
|
||||
assert_eq!(content.logprob, -9999.0);
|
||||
assert_eq!(top(&pos, 1), vec![(7, None)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_captures_late_prompt_info() {
|
||||
let stream = stream::iter(vec![
|
||||
|
||||
@@ -26,8 +26,10 @@ pub(super) struct ResponseOptions {
|
||||
pub include_usage: bool,
|
||||
/// Whether the caller asked for usage on every streamed chunk.
|
||||
pub include_continuous_usage: bool,
|
||||
/// Whether the caller requested output logprobs on generate choices.
|
||||
pub include_logprobs: bool,
|
||||
/// Requested output logprobs (`sampling_params.logprobs`): `None` omits
|
||||
/// them, `Some(n)` returns `max(n, 1)` candidates per position, `Some(-1)`
|
||||
/// all of them.
|
||||
pub logprobs: Option<i32>,
|
||||
/// Whether the caller requested top-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether the caller requested final prompt token metadata.
|
||||
@@ -56,7 +58,7 @@ pub(super) fn prepare_generate_request(
|
||||
.as_ref()
|
||||
.and_then(|options| options.continuous_usage_stats)
|
||||
.unwrap_or(false);
|
||||
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
|
||||
let logprobs = request.sampling_params.inner.logprobs;
|
||||
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
|
||||
let return_token_ids = request.return_token_ids.unwrap_or(false);
|
||||
let mut sampling_params = request.sampling_params.inner;
|
||||
@@ -96,7 +98,7 @@ pub(super) fn prepare_generate_request(
|
||||
options: ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids,
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
|
||||
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
|
||||
|
||||
/// Sampling parameters for the token-in/token-out generate API.
|
||||
///
|
||||
@@ -61,7 +61,7 @@ impl Normalizable for GenerateRequest {}
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateResponseChoice {
|
||||
pub index: u32,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub logprobs: Option<GenerateLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
@@ -72,7 +72,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateResponseStreamChoice {
|
||||
pub index: u32,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub logprobs: Option<GenerateLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
@@ -197,6 +197,32 @@ pub(super) struct PlaceholderRangeInfo {
|
||||
pub length: usize,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateLogProbs` class: output logprobs for one
|
||||
/// choice, carrying integer token ids rather than the OpenAI string token.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogProbs {
|
||||
pub content: Option<Vec<GenerateLogProbsContent>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateLogProbsContent` class: the sampled token
|
||||
/// at one position plus its top-k candidates, in the engine's order: the
|
||||
/// sampled token first, then the remaining candidates in rank order.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogProbsContent {
|
||||
pub token_id: u32,
|
||||
pub logprob: f32,
|
||||
pub rank: Option<u32>,
|
||||
pub top_logprobs: Vec<GenerateLogProb>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateLogProb` class: one candidate token.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogProb {
|
||||
pub token_id: u32,
|
||||
pub logprob: f32,
|
||||
pub rank: Option<u32>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogprob {
|
||||
|
||||
@@ -4453,13 +4453,15 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
|
||||
assert_eq!(json["choices"][0]["index"], 0);
|
||||
assert_eq!(json["choices"][0]["token_ids"], json!([33, 44]));
|
||||
assert_eq!(json["choices"][0]["finish_reason"], "stop");
|
||||
// Output logprobs carry integer token ids and rank, not the OpenAI
|
||||
// "token_id:N" placeholder string; `bytes` is gone.
|
||||
assert_eq!(json["choices"][0]["logprobs"]["content"][0]["token_id"], 33);
|
||||
assert_eq!(json["choices"][0]["logprobs"]["content"][0]["rank"], 1);
|
||||
assert!(json["choices"][0]["logprobs"]["content"][0]["token"].is_null());
|
||||
assert!(json["choices"][0]["logprobs"]["content"][0]["bytes"].is_null());
|
||||
assert_eq!(
|
||||
json["choices"][0]["logprobs"]["content"][0]["token"],
|
||||
"token_id:33"
|
||||
);
|
||||
assert_eq!(
|
||||
json["choices"][0]["logprobs"]["content"][1]["top_logprobs"][0]["token"],
|
||||
"token_id:44"
|
||||
json["choices"][0]["logprobs"]["content"][1]["top_logprobs"][0]["token_id"],
|
||||
44
|
||||
);
|
||||
assert_eq!(json["prompt_logprobs"][0], serde_json::Value::Null);
|
||||
assert_eq!(
|
||||
@@ -4581,9 +4583,11 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
|
||||
assert_eq!(first["choices"][0]["index"], 0);
|
||||
assert_eq!(first["choices"][0]["token_ids"], json!([33]));
|
||||
assert_eq!(
|
||||
first["choices"][0]["logprobs"]["content"][0]["token"],
|
||||
"token_id:33"
|
||||
first["choices"][0]["logprobs"]["content"][0]["token_id"],
|
||||
33
|
||||
);
|
||||
assert_eq!(first["choices"][0]["logprobs"]["content"][0]["rank"], 1);
|
||||
assert!(first["choices"][0]["logprobs"]["content"][0]["token"].is_null());
|
||||
assert_eq!(first["usage"]["prompt_tokens"], 2);
|
||||
assert_eq!(first["usage"]["completion_tokens"], 1);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user