[Rust Frontend] Attribute decoded text to tokens (#52910)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-01 22:25:31 +00:00
committed by GitHub
parent 259a209bfa
commit b911fe85c4
17 changed files with 1182 additions and 320 deletions
+2 -1
View File
@@ -19,7 +19,8 @@ pub use lower::{
};
pub use output::{
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
DecodedText, DecodedTextEvent, DecodedTokenLogprob, Finished, SampledDelta, TextDecodeOptions,
TextOutputStreamExt, TokenAnchor, TokenAttribution,
};
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
use trait_set::trait_set;
+182 -48
View File
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
use super::logprobs::{
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
@@ -52,6 +52,19 @@ pub struct Finished {
pub ec_transfer_params: Option<serde_json::Value>,
}
/// Sample metadata emitted by one engine output update.
///
/// Token IDs and logprobs share the engine-update clock and align by generated
/// token position. They may arrive before the corresponding decoded text is
/// visible or alongside decoded text attributed to earlier updates.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SampledDelta {
/// Token IDs produced by this engine update, after stop-string truncation.
pub token_ids: Vec<u32>,
/// Per-position logprobs aligned with `token_ids`, when requested.
pub logprobs: Option<DecodedLogprobs>,
}
/// Internal decoded-text event emitted before higher-level assistant
/// adaptation.
#[derive(Debug, Clone, PartialEq)]
@@ -68,25 +81,25 @@ pub enum DecodedTextEvent {
/// remaining prompt positions.
prompt_logprobs: Option<DecodedPromptLogprobs>,
},
/// A delta of text has been decoded, optionally alongside token-position
/// logprobs.
/// Decoded text and sampled metadata observed during one output update.
///
/// `delta` is the newly visible decoded text fragment for this update.
/// `decoded` follows the decoder clock. It may be empty while tokens remain
/// undecodable, and may later contain token attributions from earlier
/// engine updates. Tokens suppressed before decoding are absent.
///
/// `logprobs` covers the newly generated token positions from the same
/// update, but is not guaranteed to align with `delta` by character
/// span. One update may carry token logprobs but no newly visible text
/// yet, and one visible text fragment may reflect multiple token
/// positions becoming decodable together.
/// `sampled` follows the engine-update clock. Its token IDs and logprobs
/// align with each other. `decoded.attributions` has an independent decoder
/// clock and positional sequence.
///
/// Upper-level may further parse `delta` as reasoning or tool calls.
/// Upper-level may further parse `decoded.text` as reasoning or tool calls.
///
/// When `finished` is `Some`, this is the terminal event for the request.
TextDelta {
delta: String,
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
decoded: DecodedText,
sampled: SampledDelta,
/// Heap-backed because terminal metadata is large and appears once per
/// request, while every streaming event carries this enum variant.
finished: Option<Box<Finished>>,
},
}
@@ -104,9 +117,9 @@ pub async fn decoded_text_event_stream(
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
let mut prompt_token_count = 0_usize;
let mut cached_token_count = 0_usize;
let mut token_ids = Vec::new();
let mut sampled_token_ids = Vec::new();
let mut output_token_count: usize = 0;
let mut logprobs: Option<DecodedLogprobs> = None;
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
while let Some(next) = raw_stream.next().await {
let output = next?;
@@ -168,7 +181,7 @@ pub async fn decoded_text_event_stream(
&output.token_ids
};
let mut delta: Option<String> = None;
let mut decoded = DecodedText::default();
let mut truncate_output_to = None;
let mut truncate_tokens_to = None;
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
@@ -189,12 +202,9 @@ pub async fn decoded_text_event_stream(
break;
}
// TODO: avoid generating a chunk every time a token is pushed
if intermediate && let Some(chunk) = decoder.next_chunk() {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
decoded.append(chunk);
}
}
@@ -223,9 +233,9 @@ pub async fn decoded_text_event_stream(
.transpose()?;
if !intermediate {
token_ids.extend(&new_token_ids);
sampled_token_ids.extend(&new_token_ids);
if let Some(dlp) = decoded_logprobs.as_ref() {
logprobs
sampled_logprobs
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
.positions
.extend_from_slice(&dlp.positions);
@@ -234,22 +244,19 @@ pub async fn decoded_text_event_stream(
if let Some(reason) = finish_reason {
// Flush any remaining buffered text.
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
let text_len = text.len();
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
if intermediate {
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
let text_len = full_decoded.text.len();
let full_text = tracing::enabled!(Level::TRACE).then(|| full_decoded.text.clone());
let decoded = if intermediate {
if let Some(chunk) = last_chunk {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
decoded.append(chunk);
}
token_ids = new_token_ids;
logprobs = decoded_logprobs;
text = delta.unwrap_or_default();
}
sampled_token_ids = new_token_ids;
sampled_logprobs = decoded_logprobs;
decoded
} else {
full_decoded
};
debug!(
finish_reason = ?reason,
@@ -269,10 +276,12 @@ pub async fn decoded_text_event_stream(
}
y.yield_ok(DecodedTextEvent::TextDelta {
delta: text,
token_ids,
logprobs,
finished: Some(Finished {
decoded,
sampled: SampledDelta {
token_ids: sampled_token_ids,
logprobs: sampled_logprobs,
},
finished: Some(Box::new(Finished {
usage: TokenUsage {
prompt_token_count,
output_token_count,
@@ -281,7 +290,7 @@ pub async fn decoded_text_event_stream(
finish_reason: reason,
kv_transfer_params,
ec_transfer_params,
}),
})),
})
.await;
return Ok(());
@@ -289,9 +298,11 @@ pub async fn decoded_text_event_stream(
if intermediate {
y.yield_ok(DecodedTextEvent::TextDelta {
delta: delta.unwrap_or_default(),
token_ids: new_token_ids,
logprobs: decoded_logprobs,
decoded,
sampled: SampledDelta {
token_ids: new_token_ids,
logprobs: decoded_logprobs,
},
finished: None,
})
.await;
@@ -338,10 +349,10 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use futures::{Stream, stream};
use futures::{Stream, StreamExt as _, stream};
use vllm_engine_core_client::AbortCause;
use vllm_llm::GenerateOutput;
use vllm_tokenizer::test_utils::TestTokenizer;
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
use super::*;
use crate::output::TextOutputStreamExt as _;
@@ -378,6 +389,129 @@ mod tests {
}
}
fn visible(token_id: u32, byte_offset: u32) -> TokenAttribution {
TokenAttribution {
token_id,
anchor: TokenAnchor::Visible { byte_offset },
}
}
fn text_delta(
text: impl Into<String>,
attributions: impl IntoIterator<Item = TokenAttribution>,
sampled_token_ids: Vec<u32>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText {
text: text.into(),
attributions: attributions.into_iter().collect(),
},
sampled: SampledDelta {
token_ids: sampled_token_ids,
logprobs: None,
},
finished: finished.map(Box::new),
}
}
async fn run_intermediate(
outputs: Vec<GenerateOutput>,
decode_options: TextDecodeOptions,
) -> Vec<DecodedTextEvent> {
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
decoded_text_event_stream(
"test".into(),
tokenizer,
stream::iter(outputs.into_iter().map(Ok)),
decode_options,
true,
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap()
}
#[tokio::test]
async fn decoded_and_sampled_clocks_are_independent() {
struct Case {
name: &'static str,
outputs: Vec<GenerateOutput>,
expected: Vec<DecodedTextEvent>,
}
let prompt: Arc<[u32]> = Arc::from([]);
let cases = [
Case {
name: "utf8 byte fallback resolves across engine updates",
outputs: vec![
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
],
expected: vec![
DecodedTextEvent::Start {
prompt_token_ids: Arc::clone(&prompt),
prompt_logprobs: None,
},
text_delta("", [], vec![0xe4], None),
text_delta(
"你",
[visible(0xe4, 0), visible(0xbd, 0), visible(0xa0, 0)],
vec![0xbd, 0xa0],
Some(Finished {
usage: TokenUsage {
prompt_token_count: 0,
output_token_count: 3,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
}),
),
],
},
Case {
name: "terminal stop token remains sampled and bypasses decoder",
outputs: vec![GenerateOutput::for_test(
Some(Arc::clone(&prompt)),
ascii_tokens("a!"),
Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))),
)],
expected: vec![
DecodedTextEvent::Start {
prompt_token_ids: Arc::clone(&prompt),
prompt_logprobs: None,
},
text_delta(
"a",
[visible(b'a' as u32, 0)],
ascii_tokens("a!"),
Some(Finished {
usage: TokenUsage {
prompt_token_count: 0,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(
b'!' as u32,
))),
kv_transfer_params: None,
ec_transfer_params: None,
}),
),
],
},
];
for case in cases {
let actual = run_intermediate(case.outputs, TextDecodeOptions::default()).await;
assert_eq!(actual, case.expected, "{}", case.name);
}
}
struct DropRecordingStream {
next: Option<vllm_llm::Result<GenerateOutput>>,
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
+96 -84
View File
@@ -3,10 +3,13 @@
//! Output processing helpers shared by text and chat layers.
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
pub use decoded::{
DecodedTextEvent, Finished, SampledDelta, TextDecodeOptions, decoded_text_event_stream,
};
pub use logprobs::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
};
pub use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution};
mod decoded;
mod logprobs;
@@ -58,13 +61,16 @@ impl<T: TextOutputStream> T {
prompt_token_ids = start_prompt_token_ids;
}
DecodedTextEvent::TextDelta {
delta,
token_ids: delta_token_ids,
logprobs: mut delta_logprobs,
decoded,
sampled:
SampledDelta {
token_ids: delta_token_ids,
logprobs: mut delta_logprobs,
},
finished,
} => {
if let Some(c) = collected.as_mut() {
c.text.push_str(&delta);
c.text.push_str(&decoded.text);
c.token_ids.extend(delta_token_ids);
if let Some(dlp) = delta_logprobs.as_mut() {
if let Some(lp) = c.logprobs.as_mut() {
@@ -75,7 +81,7 @@ impl<T: TextOutputStream> T {
}
} else {
collected = Some(CollectedTextOutput {
text: delta,
text: decoded.text,
prompt_token_ids: Arc::clone(&prompt_token_ids),
prompt_logprobs: prompt_logprobs.take(),
logprobs: delta_logprobs,
@@ -134,29 +140,31 @@ mod tests {
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: "bc".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
decoded: DecodedText::unattributed("bc"),
sampled: SampledDelta {
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
}),
},
finished: Some(Box::new(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 2,
@@ -165,7 +173,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}),
]);
@@ -219,62 +227,66 @@ mod tests {
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "he".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
],
}),
decoded: DecodedText::unattributed("he"),
sampled: SampledDelta {
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
],
}),
},
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "llo".to_string(),
token_ids: vec![3, 4, 5],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
decoded: DecodedText::unattributed("llo"),
sampled: SampledDelta {
token_ids: vec![3, 4, 5],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
}),
},
finished: Some(Box::new(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 5,
@@ -283,7 +295,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}),
]);