[Rust Frontend] Attribute decoded text to tokens (#52910)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-01 22:25:31 +00:00
committed by GitHub
parent 259a209bfa
commit b911fe85c4
17 changed files with 1182 additions and 320 deletions
+1
View File
@@ -6043,6 +6043,7 @@ dependencies = [
"rustc-hash 1.1.0",
"serde",
"serde_json",
"smallvec",
"tekken",
"tempfile",
"thiserror 2.0.18",
+1
View File
@@ -99,6 +99,7 @@ serde_tuple = "1.1.3"
serde_with = "3.18.0"
serial_test = { version = "3.2.0", features = ["file_locks"] }
sha2 = "0.10.9"
smallvec = "1.15.1"
socket2 = "0.6.3"
strum = { version = "0.27.2", features = ["derive"] }
subenum = "1.1.3"
+14 -13
View File
@@ -13,7 +13,7 @@ use futures::{StreamExt as _, pin_mut};
use thiserror_ext::AsReport;
use tracing::warn;
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
use vllm_text::output::DecodedTextEvent;
use vllm_text::output::{DecodedTextEvent, SampledDelta};
use crate::Result;
use crate::error::Error;
@@ -237,12 +237,15 @@ pub(crate) async fn unified_event_stream(
.await;
}
DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
} => {
for next in state.process_delta(delta)? {
for next in state.process_delta(decoded.text)? {
y.yield_ok(next).await;
}
if logprobs.is_some() || !token_ids.is_empty() {
@@ -375,24 +378,22 @@ mod tests {
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
decoded: vllm_text::DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: None,
}
}
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
delta: delta.to_string(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(vllm_text::output::Finished {
decoded: vllm_text::DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: Some(Box::new(vllm_text::output::Finished {
usage: vllm_llm::TokenUsage::default(),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}
}
+7 -4
View File
@@ -11,7 +11,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
use futures::StreamExt as _;
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
use openai_harmony::{HarmonyEncoding, StreamableParser};
use vllm_text::output::DecodedTextEvent;
use vllm_text::output::{DecodedTextEvent, SampledDelta};
use crate::Result as ChatResult;
use crate::error::{Error, Result};
@@ -345,9 +345,12 @@ async fn harmony_assistant_event_stream(
.await;
}
DecodedTextEvent::TextDelta {
delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here
token_ids,
logprobs,
decoded: _, // harmony takes raw token IDs as input, so we ignore decoded text here
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
} => {
for event in state.process_token_ids(&token_ids)? {
+30 -49
View File
@@ -6,7 +6,9 @@ use std::sync::Arc;
use futures::executor::block_on;
use futures::{TryStreamExt as _, stream};
use openai_harmony::chat::{Message, Role};
use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
use vllm_text::output::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedText, DecodedTextEvent, Finished, SampledDelta,
};
use super::*;
use crate::output::ChatOutputProcessor;
@@ -59,6 +61,21 @@ fn finished() -> Finished {
}
}
fn decoded_tokens(
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText::default(),
sampled: SampledDelta {
token_ids,
logprobs,
},
finished: finished.map(Box::new),
}
}
async fn collect_events(
processor: HarmonyChatOutputProcessor,
events: Vec<DecodedTextEvent>,
@@ -96,12 +113,7 @@ fn interrupted_final_message_is_preserved() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..tokens.len() - 1].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
],
));
@@ -135,12 +147,7 @@ fn eos_flush_preserves_trailing_replacement_text() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -157,12 +164,7 @@ fn interrupted_analysis_message_is_preserved() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..tokens.len() - 1].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
],
));
@@ -196,12 +198,7 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -225,12 +222,7 @@ fn multiple_messages_get_newline_separators() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens, None, Some(finished())),
],
));
@@ -256,18 +248,8 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[..midpoint].to_vec(),
logprobs: None,
finished: None,
},
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens[midpoint..].to_vec(),
logprobs: None,
finished: Some(finished()),
},
decoded_tokens(tokens[..midpoint].to_vec(), None, None),
decoded_tokens(tokens[midpoint..].to_vec(), None, Some(finished())),
],
));
@@ -303,14 +285,13 @@ fn semantic_events_precede_same_update_logprobs() {
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
vec![
decoded_start(),
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: tokens,
logprobs: Some(DecodedLogprobs {
decoded_tokens(
tokens,
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs { entries: vec![] }],
}),
finished: Some(finished()),
},
Some(finished()),
),
],
));
+13 -12
View File
@@ -784,15 +784,14 @@ fn decoded_completion_stream(
if chunks.is_empty() {
events.push({
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(Finished {
decoded: vllm_text::DecodedText::default(),
sampled: vllm_text::SampledDelta::default(),
finished: Some(Box::new(Finished {
usage: Default::default(),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}
});
} else {
@@ -805,10 +804,12 @@ fn decoded_completion_stream(
ec_transfer_params: None,
});
events.push(DecodedTextEvent::TextDelta {
delta: chunk.delta,
token_ids: chunk.token_ids,
logprobs: None,
finished,
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
sampled: vllm_text::SampledDelta {
token_ids: chunk.token_ids,
logprobs: None,
},
finished: finished.map(Box::new),
});
}
}
@@ -864,7 +865,7 @@ fn incremental_decode_chunks(
for token_id in chunk_token_ids.iter().copied() {
decoder.push_token(token_id)?;
while let Some(chunk) = decoder.next_chunk() {
delta.push_str(&chunk);
delta.push_str(&chunk.text);
}
}
chunks.push(DecodedCompletionChunk {
@@ -876,10 +877,10 @@ fn incremental_decode_chunks(
let (last_chunk, _) = decoder.flush(None)?;
if let Some(last_chunk) = last_chunk {
if let Some(delta) = chunks.last_mut() {
delta.delta.push_str(&last_chunk);
delta.delta.push_str(&last_chunk.text);
} else {
chunks.push(DecodedCompletionChunk {
delta: last_chunk,
delta: last_chunk.text,
token_ids: Vec::new(),
});
}
+9 -6
View File
@@ -14,7 +14,7 @@ use tracing::{Span, info, info_span, warn};
use tracing_futures::Instrument as _;
use uuid::Uuid;
use vllm_llm::current_unix_timestamp_secs;
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
use vllm_text::{DecodedTextEvent, Prompt, SampledDelta, TextOutputStreamExt as _, TextRequest};
use super::convert::{self, ResponseOpts};
use super::{InferenceServer, pb};
@@ -264,17 +264,20 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
})
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
}) => Ok(pb::GenerateResponse {
prompt_info: None,
outputs: Some(convert::to_sequence_output(
&delta,
&decoded.text,
&token_ids,
logprobs.as_ref(),
finished.as_ref(),
finished.as_deref(),
&response_opts,
)),
}),
@@ -23,7 +23,7 @@ use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::tokenizer::Tokenizer;
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, SampledDelta, TextOutputStream,
TextOutputStreamExt as _, TextRequest,
};
@@ -317,11 +317,15 @@ async fn completion_chunk_stream(
}
}
Ok(DecodedTextEvent::TextDelta {
delta,
token_ids,
logprobs,
decoded,
sampled:
SampledDelta {
token_ids,
logprobs,
},
finished,
}) => {
let delta = decoded.text;
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
// The one generated token is only used to drive the engine to a finished event,
// so hide its delta and forward only the terminal finish/usage metadata.
@@ -561,8 +565,8 @@ mod tests {
use itertools::Itertools as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
DecodedTokenLogprob, FinishReason, Finished,
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedText,
DecodedTextEvent, DecodedTokenLogprob, FinishReason, Finished, SampledDelta,
};
use super::{
@@ -570,6 +574,22 @@ mod tests {
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
};
fn decoded_delta(
text: impl Into<String>,
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText::unattributed(text),
sampled: SampledDelta {
token_ids,
logprobs,
},
finished: finished.map(Box::new),
}
}
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
Arc::new(StreamResponseEnvelope::new(
"cmpl-1".to_string(),
@@ -619,10 +639,10 @@ mod tests {
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "h".to_string(),
token_ids: vec![b'h' as u32],
logprobs: Some(DecodedLogprobs {
Ok(decoded_delta(
"h",
vec![b'h' as u32],
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
@@ -640,12 +660,12 @@ mod tests {
],
}],
}),
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: vec![b'!' as u32],
logprobs: Some(DecodedLogprobs {
None,
)),
Ok(decoded_delta(
"",
vec![b'!' as u32],
Some(DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
@@ -663,7 +683,7 @@ mod tests {
],
}],
}),
finished: Some(Finished {
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 5,
output_token_count: 2,
@@ -675,7 +695,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -773,11 +793,11 @@ mod tests {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -787,7 +807,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let response = super::collect_completion(
@@ -825,11 +845,11 @@ mod tests {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
@@ -839,7 +859,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let response = super::collect_completion(
@@ -880,11 +900,11 @@ mod tests {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -894,7 +914,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -952,11 +972,11 @@ mod tests {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
@@ -966,7 +986,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
@@ -1026,11 +1046,11 @@ mod tests {
}],
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
Ok(decoded_delta(
" leaked",
vec![3],
None,
Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
@@ -1040,7 +1060,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
}),
}),
)),
]);
let chunks = completion_chunk_stream(
+2 -1
View File
@@ -19,7 +19,8 @@ pub use lower::{
};
pub use output::{
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
DecodedText, DecodedTextEvent, DecodedTokenLogprob, Finished, SampledDelta, TextDecodeOptions,
TextOutputStreamExt, TokenAnchor, TokenAttribution,
};
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
use trait_set::trait_set;
+182 -48
View File
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
use super::logprobs::{
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
@@ -52,6 +52,19 @@ pub struct Finished {
pub ec_transfer_params: Option<serde_json::Value>,
}
/// Sample metadata emitted by one engine output update.
///
/// Token IDs and logprobs share the engine-update clock and align by generated
/// token position. They may arrive before the corresponding decoded text is
/// visible or alongside decoded text attributed to earlier updates.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SampledDelta {
/// Token IDs produced by this engine update, after stop-string truncation.
pub token_ids: Vec<u32>,
/// Per-position logprobs aligned with `token_ids`, when requested.
pub logprobs: Option<DecodedLogprobs>,
}
/// Internal decoded-text event emitted before higher-level assistant
/// adaptation.
#[derive(Debug, Clone, PartialEq)]
@@ -68,25 +81,25 @@ pub enum DecodedTextEvent {
/// remaining prompt positions.
prompt_logprobs: Option<DecodedPromptLogprobs>,
},
/// A delta of text has been decoded, optionally alongside token-position
/// logprobs.
/// Decoded text and sampled metadata observed during one output update.
///
/// `delta` is the newly visible decoded text fragment for this update.
/// `decoded` follows the decoder clock. It may be empty while tokens remain
/// undecodable, and may later contain token attributions from earlier
/// engine updates. Tokens suppressed before decoding are absent.
///
/// `logprobs` covers the newly generated token positions from the same
/// update, but is not guaranteed to align with `delta` by character
/// span. One update may carry token logprobs but no newly visible text
/// yet, and one visible text fragment may reflect multiple token
/// positions becoming decodable together.
/// `sampled` follows the engine-update clock. Its token IDs and logprobs
/// align with each other. `decoded.attributions` has an independent decoder
/// clock and positional sequence.
///
/// Upper-level may further parse `delta` as reasoning or tool calls.
/// Upper-level may further parse `decoded.text` as reasoning or tool calls.
///
/// When `finished` is `Some`, this is the terminal event for the request.
TextDelta {
delta: String,
token_ids: Vec<u32>,
logprobs: Option<DecodedLogprobs>,
finished: Option<Finished>,
decoded: DecodedText,
sampled: SampledDelta,
/// Heap-backed because terminal metadata is large and appears once per
/// request, while every streaming event carries this enum variant.
finished: Option<Box<Finished>>,
},
}
@@ -104,9 +117,9 @@ pub async fn decoded_text_event_stream(
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
let mut prompt_token_count = 0_usize;
let mut cached_token_count = 0_usize;
let mut token_ids = Vec::new();
let mut sampled_token_ids = Vec::new();
let mut output_token_count: usize = 0;
let mut logprobs: Option<DecodedLogprobs> = None;
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
while let Some(next) = raw_stream.next().await {
let output = next?;
@@ -168,7 +181,7 @@ pub async fn decoded_text_event_stream(
&output.token_ids
};
let mut delta: Option<String> = None;
let mut decoded = DecodedText::default();
let mut truncate_output_to = None;
let mut truncate_tokens_to = None;
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
@@ -189,12 +202,9 @@ pub async fn decoded_text_event_stream(
break;
}
// TODO: avoid generating a chunk every time a token is pushed
if intermediate && let Some(chunk) = decoder.next_chunk() {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
decoded.append(chunk);
}
}
@@ -223,9 +233,9 @@ pub async fn decoded_text_event_stream(
.transpose()?;
if !intermediate {
token_ids.extend(&new_token_ids);
sampled_token_ids.extend(&new_token_ids);
if let Some(dlp) = decoded_logprobs.as_ref() {
logprobs
sampled_logprobs
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
.positions
.extend_from_slice(&dlp.positions);
@@ -234,22 +244,19 @@ pub async fn decoded_text_event_stream(
if let Some(reason) = finish_reason {
// Flush any remaining buffered text.
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
let text_len = text.len();
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
if intermediate {
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
let text_len = full_decoded.text.len();
let full_text = tracing::enabled!(Level::TRACE).then(|| full_decoded.text.clone());
let decoded = if intermediate {
if let Some(chunk) = last_chunk {
if let Some(delta_str) = delta.as_mut() {
delta_str.push_str(&chunk);
} else {
delta = Some(chunk);
}
decoded.append(chunk);
}
token_ids = new_token_ids;
logprobs = decoded_logprobs;
text = delta.unwrap_or_default();
}
sampled_token_ids = new_token_ids;
sampled_logprobs = decoded_logprobs;
decoded
} else {
full_decoded
};
debug!(
finish_reason = ?reason,
@@ -269,10 +276,12 @@ pub async fn decoded_text_event_stream(
}
y.yield_ok(DecodedTextEvent::TextDelta {
delta: text,
token_ids,
logprobs,
finished: Some(Finished {
decoded,
sampled: SampledDelta {
token_ids: sampled_token_ids,
logprobs: sampled_logprobs,
},
finished: Some(Box::new(Finished {
usage: TokenUsage {
prompt_token_count,
output_token_count,
@@ -281,7 +290,7 @@ pub async fn decoded_text_event_stream(
finish_reason: reason,
kv_transfer_params,
ec_transfer_params,
}),
})),
})
.await;
return Ok(());
@@ -289,9 +298,11 @@ pub async fn decoded_text_event_stream(
if intermediate {
y.yield_ok(DecodedTextEvent::TextDelta {
delta: delta.unwrap_or_default(),
token_ids: new_token_ids,
logprobs: decoded_logprobs,
decoded,
sampled: SampledDelta {
token_ids: new_token_ids,
logprobs: decoded_logprobs,
},
finished: None,
})
.await;
@@ -338,10 +349,10 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use futures::{Stream, stream};
use futures::{Stream, StreamExt as _, stream};
use vllm_engine_core_client::AbortCause;
use vllm_llm::GenerateOutput;
use vllm_tokenizer::test_utils::TestTokenizer;
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
use super::*;
use crate::output::TextOutputStreamExt as _;
@@ -378,6 +389,129 @@ mod tests {
}
}
fn visible(token_id: u32, byte_offset: u32) -> TokenAttribution {
TokenAttribution {
token_id,
anchor: TokenAnchor::Visible { byte_offset },
}
}
fn text_delta(
text: impl Into<String>,
attributions: impl IntoIterator<Item = TokenAttribution>,
sampled_token_ids: Vec<u32>,
finished: Option<Finished>,
) -> DecodedTextEvent {
DecodedTextEvent::TextDelta {
decoded: DecodedText {
text: text.into(),
attributions: attributions.into_iter().collect(),
},
sampled: SampledDelta {
token_ids: sampled_token_ids,
logprobs: None,
},
finished: finished.map(Box::new),
}
}
async fn run_intermediate(
outputs: Vec<GenerateOutput>,
decode_options: TextDecodeOptions,
) -> Vec<DecodedTextEvent> {
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
decoded_text_event_stream(
"test".into(),
tokenizer,
stream::iter(outputs.into_iter().map(Ok)),
decode_options,
true,
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap()
}
#[tokio::test]
async fn decoded_and_sampled_clocks_are_independent() {
struct Case {
name: &'static str,
outputs: Vec<GenerateOutput>,
expected: Vec<DecodedTextEvent>,
}
let prompt: Arc<[u32]> = Arc::from([]);
let cases = [
Case {
name: "utf8 byte fallback resolves across engine updates",
outputs: vec![
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
],
expected: vec![
DecodedTextEvent::Start {
prompt_token_ids: Arc::clone(&prompt),
prompt_logprobs: None,
},
text_delta("", [], vec![0xe4], None),
text_delta(
"你",
[visible(0xe4, 0), visible(0xbd, 0), visible(0xa0, 0)],
vec![0xbd, 0xa0],
Some(Finished {
usage: TokenUsage {
prompt_token_count: 0,
output_token_count: 3,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
ec_transfer_params: None,
}),
),
],
},
Case {
name: "terminal stop token remains sampled and bypasses decoder",
outputs: vec![GenerateOutput::for_test(
Some(Arc::clone(&prompt)),
ascii_tokens("a!"),
Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))),
)],
expected: vec![
DecodedTextEvent::Start {
prompt_token_ids: Arc::clone(&prompt),
prompt_logprobs: None,
},
text_delta(
"a",
[visible(b'a' as u32, 0)],
ascii_tokens("a!"),
Some(Finished {
usage: TokenUsage {
prompt_token_count: 0,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(
b'!' as u32,
))),
kv_transfer_params: None,
ec_transfer_params: None,
}),
),
],
},
];
for case in cases {
let actual = run_intermediate(case.outputs, TextDecodeOptions::default()).await;
assert_eq!(actual, case.expected, "{}", case.name);
}
}
struct DropRecordingStream {
next: Option<vllm_llm::Result<GenerateOutput>>,
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
+96 -84
View File
@@ -3,10 +3,13 @@
//! Output processing helpers shared by text and chat layers.
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
pub use decoded::{
DecodedTextEvent, Finished, SampledDelta, TextDecodeOptions, decoded_text_event_stream,
};
pub use logprobs::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
};
pub use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution};
mod decoded;
mod logprobs;
@@ -58,13 +61,16 @@ impl<T: TextOutputStream> T {
prompt_token_ids = start_prompt_token_ids;
}
DecodedTextEvent::TextDelta {
delta,
token_ids: delta_token_ids,
logprobs: mut delta_logprobs,
decoded,
sampled:
SampledDelta {
token_ids: delta_token_ids,
logprobs: mut delta_logprobs,
},
finished,
} => {
if let Some(c) = collected.as_mut() {
c.text.push_str(&delta);
c.text.push_str(&decoded.text);
c.token_ids.extend(delta_token_ids);
if let Some(dlp) = delta_logprobs.as_mut() {
if let Some(lp) = c.logprobs.as_mut() {
@@ -75,7 +81,7 @@ impl<T: TextOutputStream> T {
}
} else {
collected = Some(CollectedTextOutput {
text: delta,
text: decoded.text,
prompt_token_ids: Arc::clone(&prompt_token_ids),
prompt_logprobs: prompt_logprobs.take(),
logprobs: delta_logprobs,
@@ -134,29 +140,31 @@ mod tests {
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: "bc".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
decoded: DecodedText::unattributed("bc"),
sampled: SampledDelta {
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "a".to_string(),
logprob: -0.2,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "bc".to_string(),
logprob: -0.3,
rank: 1,
}],
},
],
}),
},
finished: Some(Box::new(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 2,
@@ -165,7 +173,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}),
]);
@@ -219,62 +227,66 @@ mod tests {
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "he".to_string(),
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
],
}),
decoded: DecodedText::unattributed("he"),
sampled: SampledDelta {
token_ids: vec![1, 2],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "h".to_string(),
logprob: -0.1,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "e".to_string(),
logprob: -0.2,
rank: 1,
}],
},
],
}),
},
finished: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: "llo".to_string(),
token_ids: vec![3, 4, 5],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
}),
finished: Some(Finished {
decoded: DecodedText::unattributed("llo"),
sampled: SampledDelta {
token_ids: vec![3, 4, 5],
logprobs: Some(DecodedLogprobs {
positions: vec![
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.3,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "l".to_string(),
logprob: -0.4,
rank: 1,
}],
},
DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 0,
token: "o".to_string(),
logprob: -0.5,
rank: 1,
}],
},
],
}),
},
finished: Some(Box::new(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 5,
@@ -283,7 +295,7 @@ mod tests {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
})),
}),
]);
+1
View File
@@ -14,6 +14,7 @@ riptoken.workspace = true
rustc-hash.workspace = true
serde.workspace = true
serde_json.workspace = true
smallvec.workspace = true
tekken.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
+63 -46
View File
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::mem::take;
use crate::{Result, Tokenizer};
mod attribution;
use attribution::AttributedTextBuffer;
pub use attribution::{DecodedText, TokenAnchor, TokenAttribution};
/// Stateful incremental decoder that emits text chunks one token at a time.
pub trait IncrementalDecoder: Send {
/// Push one generated token and return how many new string bytes were
@@ -12,13 +15,16 @@ pub trait IncrementalDecoder: Send {
fn push_token(&mut self, token_id: u32) -> Result<usize>;
/// Consume any text which is currently ready.
fn next_chunk(&mut self) -> Option<String>;
fn next_chunk(&mut self) -> Option<DecodedText>;
/// Flush any remaining buffered text that has not yet been emitted.
///
/// Called after the final generated token to force out buffered/incomplete
/// fragments.
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
fn flush(
&mut self,
truncate_output_to: Option<usize>,
) -> Result<(Option<DecodedText>, DecodedText)>;
/// Return cumulative decoded text so far.
fn output(&self) -> &str;
@@ -36,8 +42,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
prefix: String,
prefix_index: usize,
prefix_seeded: bool,
cumulative_output: String,
output_index: usize,
decoded: AttributedTextBuffer,
}
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
@@ -55,8 +60,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
prefix: String::new(),
prefix_index: 0,
prefix_seeded: prompt_token_ids.is_empty(),
cumulative_output: String::new(),
output_index: 0,
decoded: AttributedTextBuffer::default(),
}
}
}
@@ -68,6 +72,14 @@ const SAFE_SUFFIX_MIN: usize = 4;
const SAFE_SUFFIX_MAX: usize = 6;
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
fn record_token(&mut self, token_id: u32) {
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
self.decoded.record_zero_width_token(token_id);
} else {
self.decoded.record_pending_token(token_id);
}
}
/// Decode prompt-only context for prefix seeding.
///
/// Prompt ids may come from the model vocabulary rather than the local
@@ -132,6 +144,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
self.ids.push(token_id);
self.record_token(token_id);
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
@@ -139,49 +152,48 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
// Ensure we split at a utf-8 char boundary.
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
self.cumulative_output.push_str(new_chunk);
self.decoded.append_visible_text(new_chunk);
self.ids.drain(..self.prefix_index);
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
self.prefix_index = self.ids.len();
Ok(new_chunk.len())
}
fn next_chunk(&mut self) -> Option<String> {
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
fn next_chunk(&mut self) -> Option<DecodedText> {
let cutoff = self.decoded.len().saturating_sub(self.min_bytes_to_buffer);
// Ensure we split at a utf-8 char boundary.
let cutoff = self.cumulative_output.floor_char_boundary(cutoff);
(cutoff > self.output_index).then(|| {
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
self.output_index = cutoff;
chunk
})
let cutoff = self.decoded.text().floor_char_boundary(cutoff);
self.decoded.take_ready(cutoff)
}
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
fn flush(
&mut self,
truncate_output_to: Option<usize>,
) -> Result<(Option<DecodedText>, DecodedText)> {
// If the prefix was never seeded (no push_token was called), `ids`
// holds only prompt context — decoding it would re-emit prompt text.
if self.prefix_seeded && !self.ids.is_empty() {
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
// Ensure we split at a utf-8 char boundary.
self.cumulative_output
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
if !new_chunk.is_empty() {
self.decoded.append_visible_text(new_chunk);
}
}
self.decoded.resolve_pending_zero_width();
self.ids.clear();
self.prefix.clear();
self.prefix_index = 0;
self.prefix_seeded = true;
if let Some(truncate_output_to) = truncate_output_to {
self.cumulative_output.truncate(truncate_output_to);
self.decoded.truncate(truncate_output_to);
}
let last_chunk = (self.output_index < self.cumulative_output.len())
.then(|| self.cumulative_output[self.output_index..].to_string());
self.output_index = 0;
Ok((last_chunk, take(&mut self.cumulative_output)))
Ok(self.decoded.finish())
}
fn output(&self) -> &str {
&self.cumulative_output
self.decoded.text()
}
}
@@ -245,13 +257,13 @@ mod tests {
let mut decoder = backend.create_decode_stream(&[], false, 0);
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
assert_eq!(decoder.next_chunk().unwrap().text, "o");
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
assert_eq!(decoder.next_chunk().unwrap().text, "k");
// All text already consumed via next_chunk
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "ok");
assert_eq!(full_text.text, "ok");
}
#[test]
@@ -288,6 +300,7 @@ mod tests {
0 if !skip_special_tokens => text.push_str("<special>"),
0 => {}
1 => text.push('a'),
2 => text.push('b'),
_ => {}
}
}
@@ -301,6 +314,10 @@ mod tests {
fn id_to_token(&self, _id: u32) -> Option<String> {
unreachable!()
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == 0
}
}
#[test]
@@ -393,17 +410,17 @@ mod tests {
let mut decoder = backend.create_decode_stream(&[], false, 0);
let input = b"Hello, world!";
let mut full = String::new();
let mut out = String::new();
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
full.push_str(&chunk);
out.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None); // all consumed via next_chunk
assert_eq!(full, "Hello, world!");
assert_eq!(full_text, "Hello, world!");
assert_eq!(out, "Hello, world!");
assert_eq!(full.text, "Hello, world!");
}
/// Backend simulating non-monotonic decode where adding a token changes how
@@ -470,15 +487,15 @@ mod tests {
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
chunks.push_str(&chunk);
chunks.push_str(&chunk.text);
}
}
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
assert_eq!(chunks, "Hel");
// Flush returns the rest
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk.as_deref(), Some("lo!"));
assert_eq!(full_text, "Hello!");
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk.unwrap().text, "lo!");
assert_eq!(full.text, "Hello!");
}
#[test]
@@ -493,14 +510,14 @@ mod tests {
for byte in "你好A".bytes() {
decoder.push_token(u32::from(byte)).unwrap();
if let Some(chunk) = decoder.next_chunk() {
out.push_str(&chunk);
out.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
out.push_str(&chunk);
out.push_str(&chunk.text);
}
assert_eq!(full_text, "你好A");
assert_eq!(full.text, "你好A");
assert_eq!(out, "你好A");
}
@@ -515,9 +532,9 @@ mod tests {
.collect();
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
assert_eq!(full.text, "");
}
#[test]
@@ -526,8 +543,8 @@ mod tests {
let prompt = vec![0xe4, 0xbd];
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
assert_eq!(full.text, "");
}
}
@@ -0,0 +1,261 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::mem::take;
use smallvec::SmallVec;
/// Position of one generated token in decoded text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TokenAnchor {
/// The token contributed to visible decoded text.
Visible { byte_offset: u32 },
/// The token produced no visible decoded bytes.
ZeroWidth { byte_offset: u32 },
}
impl TokenAnchor {
fn byte_offset(self) -> u32 {
match self {
Self::Visible { byte_offset } | Self::ZeroWidth { byte_offset } => byte_offset,
}
}
/// Return a new anchor offset by `delta` bytes.
#[must_use]
pub(super) fn offset_by(self, delta: i64) -> Self {
let byte_offset = i64::from(self.byte_offset())
.checked_add(delta)
.and_then(|offset| u32::try_from(offset).ok())
.expect("token anchor byte offset out of range");
match self {
Self::Visible { .. } => Self::Visible { byte_offset },
Self::ZeroWidth { .. } => Self::ZeroWidth { byte_offset },
}
}
}
/// One generated token and its position in decoded text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TokenAttribution {
/// Token ID passed to [`super::IncrementalDecoder::push_token`].
pub token_id: u32,
/// Position of the token in the decoded text.
pub anchor: TokenAnchor,
}
/// Decoded text and the generated tokens attributed to it.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct DecodedText {
/// Decoded UTF-8 text.
pub text: String,
/// One entry per generated token, in generation order.
///
/// Offsets are local to `text`. Visible offsets may repeat when multiple
/// tokens jointly decode into one character. A zero-width offset may equal
/// `text.len()`.
/// Four inline records cover one complete UTF-8 byte-fallback sequence.
pub attributions: SmallVec<[TokenAttribution; 4]>,
}
impl DecodedText {
/// Create decoded text without token attribution records. Mostly for testing purposes.
pub fn unattributed(text: impl Into<String>) -> Self {
Self {
text: text.into(),
attributions: SmallVec::new(),
}
}
/// Append another decoded fragment, rebasing its token anchors.
pub fn append(&mut self, other: Self) {
if self.text.is_empty() && self.attributions.is_empty() {
*self = other;
return;
}
let byte_offset = offset_as_u32(self.text.len());
let other_len = offset_as_u32(other.text.len());
let _combined_len = byte_offset.checked_add(other_len).expect("decoded text exceeds 4 GiB");
self.text.push_str(&other.text);
self.attributions.extend(other.attributions.into_iter().map(|attribution| {
TokenAttribution {
token_id: attribution.token_id,
anchor: attribution.anchor.offset_by(i64::from(byte_offset)),
}
}));
}
}
#[derive(Clone, Copy, Debug)]
enum PendingAnchor {
Unresolved,
Resolved(TokenAnchor),
}
#[derive(Clone, Copy, Debug)]
struct PendingAttribution {
token_id: u32,
anchor: PendingAnchor,
}
/// Cumulative decoded text plus the attribution state needed to emit local
/// chunks.
#[derive(Default)]
pub(super) struct AttributedTextBuffer {
text: String,
emitted_byte_offset: usize,
attributions: Vec<PendingAttribution>,
emitted_attribution_count: usize,
pending_start: Option<usize>,
}
impl AttributedTextBuffer {
/// Return all decoded text accumulated so far.
pub(super) fn text(&self) -> &str {
&self.text
}
/// Return the decoded text length in bytes.
pub(super) fn len(&self) -> usize {
self.text.len()
}
/// Record a token awaiting visible decoded output.
pub(super) fn record_pending_token(&mut self, token_id: u32) {
self.pending_start.get_or_insert(self.attributions.len());
self.attributions.push(PendingAttribution {
token_id,
anchor: PendingAnchor::Unresolved,
});
}
/// Record a token that produced no visible decoded output.
pub(super) fn record_zero_width_token(&mut self, token_id: u32) {
self.attributions.push(PendingAttribution {
token_id,
anchor: PendingAnchor::Resolved(TokenAnchor::ZeroWidth {
byte_offset: offset_as_u32(self.text.len()),
}),
});
}
/// Append visible text and anchor pending tokens at its start.
pub(super) fn append_visible_text(&mut self, text: &str) {
let byte_offset = offset_as_u32(self.text.len());
if let Some(pending_start) = self.pending_start.take() {
for attribution in &mut self.attributions[pending_start..] {
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
attribution.anchor =
PendingAnchor::Resolved(TokenAnchor::Visible { byte_offset });
}
}
}
self.text.push_str(text);
}
/// Resolve remaining pending tokens at the current text boundary.
pub(super) fn resolve_pending_zero_width(&mut self) {
let Some(pending_start) = self.pending_start.take() else {
return;
};
let byte_offset = offset_as_u32(self.text.len());
for attribution in &mut self.attributions[pending_start..] {
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
attribution.anchor =
PendingAnchor::Resolved(TokenAnchor::ZeroWidth { byte_offset });
}
}
}
/// Truncate text and collapse removed anchors to the new boundary.
pub(super) fn truncate(&mut self, byte_offset: usize) {
self.text.truncate(byte_offset);
let byte_offset = offset_as_u32(byte_offset);
for attribution in &mut self.attributions {
let PendingAnchor::Resolved(anchor) = &mut attribution.anchor else {
continue;
};
if anchor.byte_offset() >= byte_offset {
*anchor = TokenAnchor::ZeroWidth { byte_offset };
}
}
}
/// Emit decoded text and attributions ready through `cutoff`.
pub(super) fn take_ready(&mut self, cutoff: usize) -> Option<DecodedText> {
let chunk_start = self.emitted_byte_offset;
let cutoff_u32 = offset_as_u32(cutoff);
let mut attribution_end = self.emitted_attribution_count;
for attribution in &self.attributions[self.emitted_attribution_count..] {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
break;
};
let ready = match anchor {
TokenAnchor::Visible { byte_offset } => byte_offset < cutoff_u32,
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= cutoff_u32,
};
if !ready {
break;
}
attribution_end += 1;
}
if cutoff == chunk_start && attribution_end == self.emitted_attribution_count {
return None;
}
let chunk_start_u32 = offset_as_u32(chunk_start);
let attributions = self.attributions[self.emitted_attribution_count..attribution_end]
.iter()
.map(|attribution| {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
unreachable!("ready attributions must be resolved")
};
TokenAttribution {
token_id: attribution.token_id,
anchor: anchor.offset_by(-i64::from(chunk_start_u32)),
}
})
.collect();
let text = self.text[chunk_start..cutoff].to_string();
self.emitted_byte_offset = cutoff;
self.emitted_attribution_count = attribution_end;
Some(DecodedText { text, attributions })
}
/// Emit the final chunk and take the complete decoded output.
pub(super) fn finish(&mut self) -> (Option<DecodedText>, DecodedText) {
let last_chunk = self.take_ready(self.text.len());
let attributions = take(&mut self.attributions)
.into_iter()
.map(|attribution| {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
unreachable!("finish requires every attribution to be resolved")
};
TokenAttribution {
token_id: attribution.token_id,
anchor,
}
})
.collect();
let full = DecodedText {
text: take(&mut self.text),
attributions,
};
self.emitted_byte_offset = 0;
self.emitted_attribution_count = 0;
self.pending_start = None;
(last_chunk, full)
}
}
fn offset_as_u32(offset: usize) -> u32 {
u32::try_from(offset).expect("decoded text exceeds 4 GiB")
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,425 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use super::*;
use crate::{Result, Tokenizer, test_utils::TestTokenizer};
const SPECIAL_TOKEN_ID: u32 = 0x100;
/// Backend with tokens that decode to whole text pieces.
#[derive(Debug)]
struct PieceBackend;
impl Tokenizer for PieceBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
Ok(token_ids
.iter()
.map(|token_id| match token_id {
1 => "abcd",
2 => "ab",
3 => "<stop>",
4 => "",
_ => unreachable!("unexpected token id: {token_id}"),
})
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
}
/// Raw-byte backend with one filterable special token.
#[derive(Debug)]
struct Utf8SpecialBackend;
impl Tokenizer for Utf8SpecialBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let mut bytes = Vec::new();
for &token_id in token_ids {
if token_id == SPECIAL_TOKEN_ID {
if !skip_special_tokens {
bytes.extend_from_slice(b"<special>");
}
} else {
bytes.push(token_id as u8);
}
}
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == SPECIAL_TOKEN_ID
}
}
fn tokenizer_with_special_token() -> TestTokenizer {
TestTokenizer::new().with_special_token("<special>", SPECIAL_TOKEN_ID)
}
fn visible(byte_offset: u32) -> TokenAnchor {
TokenAnchor::Visible { byte_offset }
}
fn zero_width(byte_offset: u32) -> TokenAnchor {
TokenAnchor::ZeroWidth { byte_offset }
}
fn decoded(text: &str, token_ids: &[u32], anchors: &[TokenAnchor]) -> DecodedText {
assert_eq!(token_ids.len(), anchors.len());
DecodedText {
text: text.to_string(),
attributions: token_ids
.iter()
.copied()
.zip(anchors.iter().copied())
.map(|(token_id, anchor)| TokenAttribution { token_id, anchor })
.collect(),
}
}
#[test]
fn token_anchor_offset_by_preserves_anchor_kind() {
for (anchor, delta, expected) in [
(visible(3), -2, visible(1)),
(zero_width(1), 4, zero_width(5)),
] {
assert_eq!(anchor.offset_by(delta), expected);
}
}
#[test]
fn decoded_text_append_rebases_attributions() {
let mut combined = decoded("a", &[1], &[visible(0)]);
combined.append(decoded(
"你",
&[2, 3, 4, 5],
&[visible(0), visible(0), visible(0), zero_width(3)],
));
assert_eq!(
combined,
decoded(
"a你",
&[1, 2, 3, 4, 5],
&[
visible(0),
visible(1),
visible(1),
visible(1),
zero_width(4),
],
)
);
}
#[test]
fn decoded_text_append_moves_allocations_into_empty_receiver() {
let source = decoded(
"hello",
&[1, 2, 3, 4, 5],
&[visible(0), visible(1), visible(2), visible(3), visible(4)],
);
let text_ptr = source.text.as_ptr();
let attributions_ptr = source.attributions.as_ptr();
let mut combined = DecodedText::default();
combined.append(source);
assert_eq!(combined.text.as_ptr(), text_ptr);
assert_eq!(combined.attributions.as_ptr(), attributions_ptr);
}
#[test]
fn decoded_text_append_preserves_zero_width_only_receiver() {
let mut combined = decoded("", &[1], &[zero_width(0)]);
combined.append(decoded("a", &[2], &[visible(0)]));
assert_eq!(
combined,
decoded("a", &[1, 2], &[zero_width(0), visible(0)])
);
}
struct AttributionCase<'a> {
tokenizer: &'a dyn Tokenizer,
prompt_token_ids: Vec<u32>,
token_ids: Vec<u32>,
skip_special_tokens: bool,
min_bytes_to_buffer: usize,
truncate_output_to: Option<usize>,
expected_chunks: Vec<DecodedText>,
expected_full: DecodedText,
}
fn run_attribution_case(case: AttributionCase<'_>) {
let mut decoder = case.tokenizer.create_decode_stream(
&case.prompt_token_ids,
case.skip_special_tokens,
case.min_bytes_to_buffer,
);
let mut chunks = Vec::new();
for token_id in &case.token_ids {
decoder.push_token(*token_id).unwrap();
while let Some(chunk) = decoder.next_chunk() {
chunks.push(chunk);
}
}
let (remaining, full) = decoder.flush(case.truncate_output_to).unwrap();
if let Some(remaining) = remaining {
chunks.push(remaining);
}
assert_eq!(chunks, case.expected_chunks, "chunks");
assert_eq!(full, case.expected_full, "full output");
assert_eq!(
full.attributions.len(),
case.token_ids.len(),
"one attribution per pushed token"
);
assert_eq!(
full.attributions
.iter()
.map(|attribution| attribution.token_id)
.collect::<Vec<_>>(),
case.token_ids,
"attribution preserves pushed token IDs"
);
let mut reconstructed_text = String::new();
let mut reconstructed_attributions = Vec::new();
for chunk in &chunks {
let chunk_start = reconstructed_text.len() as u32;
reconstructed_text.push_str(&chunk.text);
reconstructed_attributions.extend(chunk.attributions.iter().map(|attribution| {
TokenAttribution {
token_id: attribution.token_id,
anchor: attribution.anchor.offset_by(i64::from(chunk_start)),
}
}));
}
assert_eq!(
reconstructed_text, full.text,
"chunk text reconstructs full text"
);
assert_eq!(
reconstructed_attributions.as_slice(),
full.attributions.as_slice(),
"chunk attributions reconstruct full attributions"
);
}
#[test]
fn attribution_ascii_tokens_have_one_visible_anchor_each() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![b'o' as u32, b'k' as u32],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![
decoded("o", &[b'o' as u32], &[visible(0)]),
decoded("k", &[b'k' as u32], &[visible(0)]),
],
expected_full: decoded("ok", &[b'o' as u32, b'k' as u32], &[visible(0), visible(1)]),
});
}
#[test]
fn attribution_byte_fallback_tokens_share_one_visible_anchor() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![0xe4, 0xbd, 0xa0],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded(
"你",
&[0xe4, 0xbd, 0xa0],
&[visible(0), visible(0), visible(0)],
)],
expected_full: decoded(
"你",
&[0xe4, 0xbd, 0xa0],
&[visible(0), visible(0), visible(0)],
),
});
}
#[test]
fn attribution_holdback_splits_text_without_repeating_token() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![1],
skip_special_tokens: false,
min_bytes_to_buffer: 2,
truncate_output_to: None,
expected_chunks: vec![decoded("ab", &[1], &[visible(0)]), decoded("cd", &[], &[])],
expected_full: decoded("abcd", &[1], &[visible(0)]),
});
}
#[test]
fn attribution_filtered_special_token_is_zero_width_at_its_byte_boundary() {
let tokenizer = tokenizer_with_special_token();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
skip_special_tokens: true,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![
decoded("a", &[b'a' as u32], &[visible(0)]),
decoded("", &[SPECIAL_TOKEN_ID], &[zero_width(0)]),
decoded("b", &[b'b' as u32], &[visible(0)]),
],
expected_full: decoded(
"ab",
&[b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
&[visible(0), zero_width(1), visible(1)],
),
});
}
#[test]
fn attribution_zero_width_preserves_order_inside_byte_fallback_group() {
run_attribution_case(AttributionCase {
tokenizer: &Utf8SpecialBackend,
prompt_token_ids: vec![],
token_ids: vec![0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
skip_special_tokens: true,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded(
"你",
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
&[visible(0), zero_width(0), visible(0), visible(0)],
)],
expected_full: decoded(
"你",
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
&[visible(0), zero_width(0), visible(0), visible(0)],
),
});
}
#[test]
fn attribution_retained_special_token_has_visible_anchor() {
run_attribution_case(AttributionCase {
tokenizer: &Utf8SpecialBackend,
prompt_token_ids: vec![],
token_ids: vec![SPECIAL_TOKEN_ID],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)])],
expected_full: decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)]),
});
}
#[test]
fn attribution_empty_decode_resolves_to_zero_width_on_flush() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![4],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("", &[4], &[zero_width(0)])],
expected_full: decoded("", &[4], &[zero_width(0)]),
});
}
#[test]
fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![0xe4, 0xbd],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)])],
expected_full: decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)]),
});
}
#[test]
fn attribution_truncation_converts_fully_removed_token_to_zero_width() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![2, 3],
skip_special_tokens: false,
min_bytes_to_buffer: 32,
truncate_output_to: Some(2),
expected_chunks: vec![decoded("ab", &[2, 3], &[visible(0), zero_width(2)])],
expected_full: decoded("ab", &[2, 3], &[visible(0), zero_width(2)]),
});
}
#[test]
fn attribution_truncation_inside_token_retains_first_byte_anchor() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![1],
skip_special_tokens: false,
min_bytes_to_buffer: 32,
truncate_output_to: Some(2),
expected_chunks: vec![decoded("ab", &[1], &[visible(0)])],
expected_full: decoded("ab", &[1], &[visible(0)]),
});
}
#[test]
fn attribution_excludes_prompt_context() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![b'H' as u32, b'i' as u32],
token_ids: vec![b'!' as u32],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("!", &[b'!' as u32], &[visible(0)])],
expected_full: decoded("!", &[b'!' as u32], &[visible(0)]),
});
}
+1 -1
View File
@@ -17,7 +17,7 @@ mod tiktoken;
pub use error::{Result, TokenizerError};
pub use hf::HuggingFaceTokenizer;
pub use incremental::IncrementalDecoder;
pub use incremental::{DecodedText, IncrementalDecoder, TokenAnchor, TokenAttribution};
pub use tekken::TekkenTokenizer;
pub use tiktoken::TiktokenTokenizer;
+8 -8
View File
@@ -964,16 +964,16 @@ mod tests {
for &id in &ids {
decoder.push_token(id).unwrap();
if let Some(chunk) = decoder.next_chunk() {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
assert_eq!(output, text);
assert_eq!(full_text, text);
assert_eq!(full.text, text);
}
}
@@ -991,16 +991,16 @@ mod tests {
for &id in &ids {
decoder.push_token(id).unwrap();
if let Some(chunk) = decoder.next_chunk() {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
assert_eq!(output, text);
assert_eq!(full_text, text);
assert_eq!(full.text, text);
}
}