[Rust Frontend] Attribute decoded text to tokens (#52910)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -19,7 +19,8 @@ pub use lower::{
|
||||
};
|
||||
pub use output::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
|
||||
DecodedText, DecodedTextEvent, DecodedTokenLogprob, Finished, SampledDelta, TextDecodeOptions,
|
||||
TextOutputStreamExt, TokenAnchor, TokenAttribution,
|
||||
};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
|
||||
use trait_set::trait_set;
|
||||
|
||||
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
|
||||
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
|
||||
|
||||
use super::logprobs::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
|
||||
@@ -52,6 +52,19 @@ pub struct Finished {
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Sample metadata emitted by one engine output update.
|
||||
///
|
||||
/// Token IDs and logprobs share the engine-update clock and align by generated
|
||||
/// token position. They may arrive before the corresponding decoded text is
|
||||
/// visible or alongside decoded text attributed to earlier updates.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct SampledDelta {
|
||||
/// Token IDs produced by this engine update, after stop-string truncation.
|
||||
pub token_ids: Vec<u32>,
|
||||
/// Per-position logprobs aligned with `token_ids`, when requested.
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
}
|
||||
|
||||
/// Internal decoded-text event emitted before higher-level assistant
|
||||
/// adaptation.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -68,25 +81,25 @@ pub enum DecodedTextEvent {
|
||||
/// remaining prompt positions.
|
||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
},
|
||||
/// A delta of text has been decoded, optionally alongside token-position
|
||||
/// logprobs.
|
||||
/// Decoded text and sampled metadata observed during one output update.
|
||||
///
|
||||
/// `delta` is the newly visible decoded text fragment for this update.
|
||||
/// `decoded` follows the decoder clock. It may be empty while tokens remain
|
||||
/// undecodable, and may later contain token attributions from earlier
|
||||
/// engine updates. Tokens suppressed before decoding are absent.
|
||||
///
|
||||
/// `logprobs` covers the newly generated token positions from the same
|
||||
/// update, but is not guaranteed to align with `delta` by character
|
||||
/// span. One update may carry token logprobs but no newly visible text
|
||||
/// yet, and one visible text fragment may reflect multiple token
|
||||
/// positions becoming decodable together.
|
||||
/// `sampled` follows the engine-update clock. Its token IDs and logprobs
|
||||
/// align with each other. `decoded.attributions` has an independent decoder
|
||||
/// clock and positional sequence.
|
||||
///
|
||||
/// Upper-level may further parse `delta` as reasoning or tool calls.
|
||||
/// Upper-level may further parse `decoded.text` as reasoning or tool calls.
|
||||
///
|
||||
/// When `finished` is `Some`, this is the terminal event for the request.
|
||||
TextDelta {
|
||||
delta: String,
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
decoded: DecodedText,
|
||||
sampled: SampledDelta,
|
||||
/// Heap-backed because terminal metadata is large and appears once per
|
||||
/// request, while every streaming event carries this enum variant.
|
||||
finished: Option<Box<Finished>>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -104,9 +117,9 @@ pub async fn decoded_text_event_stream(
|
||||
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
|
||||
let mut prompt_token_count = 0_usize;
|
||||
let mut cached_token_count = 0_usize;
|
||||
let mut token_ids = Vec::new();
|
||||
let mut sampled_token_ids = Vec::new();
|
||||
let mut output_token_count: usize = 0;
|
||||
let mut logprobs: Option<DecodedLogprobs> = None;
|
||||
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
|
||||
|
||||
while let Some(next) = raw_stream.next().await {
|
||||
let output = next?;
|
||||
@@ -168,7 +181,7 @@ pub async fn decoded_text_event_stream(
|
||||
&output.token_ids
|
||||
};
|
||||
|
||||
let mut delta: Option<String> = None;
|
||||
let mut decoded = DecodedText::default();
|
||||
let mut truncate_output_to = None;
|
||||
let mut truncate_tokens_to = None;
|
||||
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
|
||||
@@ -189,12 +202,9 @@ pub async fn decoded_text_event_stream(
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: avoid generating a chunk every time a token is pushed
|
||||
if intermediate && let Some(chunk) = decoder.next_chunk() {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
decoded.append(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,9 +233,9 @@ pub async fn decoded_text_event_stream(
|
||||
.transpose()?;
|
||||
|
||||
if !intermediate {
|
||||
token_ids.extend(&new_token_ids);
|
||||
sampled_token_ids.extend(&new_token_ids);
|
||||
if let Some(dlp) = decoded_logprobs.as_ref() {
|
||||
logprobs
|
||||
sampled_logprobs
|
||||
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
|
||||
.positions
|
||||
.extend_from_slice(&dlp.positions);
|
||||
@@ -234,22 +244,19 @@ pub async fn decoded_text_event_stream(
|
||||
|
||||
if let Some(reason) = finish_reason {
|
||||
// Flush any remaining buffered text.
|
||||
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = text.len();
|
||||
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
|
||||
|
||||
if intermediate {
|
||||
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = full_decoded.text.len();
|
||||
let full_text = tracing::enabled!(Level::TRACE).then(|| full_decoded.text.clone());
|
||||
let decoded = if intermediate {
|
||||
if let Some(chunk) = last_chunk {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
decoded.append(chunk);
|
||||
}
|
||||
token_ids = new_token_ids;
|
||||
logprobs = decoded_logprobs;
|
||||
text = delta.unwrap_or_default();
|
||||
}
|
||||
sampled_token_ids = new_token_ids;
|
||||
sampled_logprobs = decoded_logprobs;
|
||||
decoded
|
||||
} else {
|
||||
full_decoded
|
||||
};
|
||||
|
||||
debug!(
|
||||
finish_reason = ?reason,
|
||||
@@ -269,10 +276,12 @@ pub async fn decoded_text_event_stream(
|
||||
}
|
||||
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: text,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished: Some(Finished {
|
||||
decoded,
|
||||
sampled: SampledDelta {
|
||||
token_ids: sampled_token_ids,
|
||||
logprobs: sampled_logprobs,
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
@@ -281,7 +290,7 @@ pub async fn decoded_text_event_stream(
|
||||
finish_reason: reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
}),
|
||||
})),
|
||||
})
|
||||
.await;
|
||||
return Ok(());
|
||||
@@ -289,9 +298,11 @@ pub async fn decoded_text_event_stream(
|
||||
|
||||
if intermediate {
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: delta.unwrap_or_default(),
|
||||
token_ids: new_token_ids,
|
||||
logprobs: decoded_logprobs,
|
||||
decoded,
|
||||
sampled: SampledDelta {
|
||||
token_ids: new_token_ids,
|
||||
logprobs: decoded_logprobs,
|
||||
},
|
||||
finished: None,
|
||||
})
|
||||
.await;
|
||||
@@ -338,10 +349,10 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::{Stream, stream};
|
||||
use futures::{Stream, StreamExt as _, stream};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_llm::GenerateOutput;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
|
||||
|
||||
use super::*;
|
||||
use crate::output::TextOutputStreamExt as _;
|
||||
@@ -378,6 +389,129 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn visible(token_id: u32, byte_offset: u32) -> TokenAttribution {
|
||||
TokenAttribution {
|
||||
token_id,
|
||||
anchor: TokenAnchor::Visible { byte_offset },
|
||||
}
|
||||
}
|
||||
|
||||
fn text_delta(
|
||||
text: impl Into<String>,
|
||||
attributions: impl IntoIterator<Item = TokenAttribution>,
|
||||
sampled_token_ids: Vec<u32>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText {
|
||||
text: text.into(),
|
||||
attributions: attributions.into_iter().collect(),
|
||||
},
|
||||
sampled: SampledDelta {
|
||||
token_ids: sampled_token_ids,
|
||||
logprobs: None,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_intermediate(
|
||||
outputs: Vec<GenerateOutput>,
|
||||
decode_options: TextDecodeOptions,
|
||||
) -> Vec<DecodedTextEvent> {
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
decoded_text_event_stream(
|
||||
"test".into(),
|
||||
tokenizer,
|
||||
stream::iter(outputs.into_iter().map(Ok)),
|
||||
decode_options,
|
||||
true,
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decoded_and_sampled_clocks_are_independent() {
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
outputs: Vec<GenerateOutput>,
|
||||
expected: Vec<DecodedTextEvent>,
|
||||
}
|
||||
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let cases = [
|
||||
Case {
|
||||
name: "utf8 byte fallback resolves across engine updates",
|
||||
outputs: vec![
|
||||
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
|
||||
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
|
||||
],
|
||||
expected: vec![
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids: Arc::clone(&prompt),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
text_delta("", [], vec![0xe4], None),
|
||||
text_delta(
|
||||
"你",
|
||||
[visible(0xe4, 0), visible(0xbd, 0), visible(0xa0, 0)],
|
||||
vec![0xbd, 0xa0],
|
||||
Some(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 3,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
},
|
||||
Case {
|
||||
name: "terminal stop token remains sampled and bypasses decoder",
|
||||
outputs: vec![GenerateOutput::for_test(
|
||||
Some(Arc::clone(&prompt)),
|
||||
ascii_tokens("a!"),
|
||||
Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))),
|
||||
)],
|
||||
expected: vec![
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids: Arc::clone(&prompt),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
text_delta(
|
||||
"a",
|
||||
[visible(b'a' as u32, 0)],
|
||||
ascii_tokens("a!"),
|
||||
Some(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(
|
||||
b'!' as u32,
|
||||
))),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
for case in cases {
|
||||
let actual = run_intermediate(case.outputs, TextDecodeOptions::default()).await;
|
||||
assert_eq!(actual, case.expected, "{}", case.name);
|
||||
}
|
||||
}
|
||||
|
||||
struct DropRecordingStream {
|
||||
next: Option<vllm_llm::Result<GenerateOutput>>,
|
||||
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
|
||||
|
||||
@@ -3,10 +3,13 @@
|
||||
|
||||
//! Output processing helpers shared by text and chat layers.
|
||||
|
||||
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
|
||||
pub use decoded::{
|
||||
DecodedTextEvent, Finished, SampledDelta, TextDecodeOptions, decoded_text_event_stream,
|
||||
};
|
||||
pub use logprobs::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
|
||||
};
|
||||
pub use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution};
|
||||
|
||||
mod decoded;
|
||||
mod logprobs;
|
||||
@@ -58,13 +61,16 @@ impl<T: TextOutputStream> T {
|
||||
prompt_token_ids = start_prompt_token_ids;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids: delta_token_ids,
|
||||
logprobs: mut delta_logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids: delta_token_ids,
|
||||
logprobs: mut delta_logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
if let Some(c) = collected.as_mut() {
|
||||
c.text.push_str(&delta);
|
||||
c.text.push_str(&decoded.text);
|
||||
c.token_ids.extend(delta_token_ids);
|
||||
if let Some(dlp) = delta_logprobs.as_mut() {
|
||||
if let Some(lp) = c.logprobs.as_mut() {
|
||||
@@ -75,7 +81,7 @@ impl<T: TextOutputStream> T {
|
||||
}
|
||||
} else {
|
||||
collected = Some(CollectedTextOutput {
|
||||
text: delta,
|
||||
text: decoded.text,
|
||||
prompt_token_ids: Arc::clone(&prompt_token_ids),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
logprobs: delta_logprobs,
|
||||
@@ -134,29 +140,31 @@ mod tests {
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "bc".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
decoded: DecodedText::unattributed("bc"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
@@ -165,7 +173,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -219,62 +227,66 @@ mod tests {
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "he".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
decoded: DecodedText::unattributed("he"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "llo".to_string(),
|
||||
token_ids: vec![3, 4, 5],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
decoded: DecodedText::unattributed("llo"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![3, 4, 5],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 5,
|
||||
@@ -283,7 +295,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user