diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 344acd6543..29b3eb268a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6043,6 +6043,7 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "serde_json", + "smallvec", "tekken", "tempfile", "thiserror 2.0.18", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 695458a885..5c76f09855 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -99,6 +99,7 @@ serde_tuple = "1.1.3" serde_with = "3.18.0" serial_test = { version = "3.2.0", features = ["file_locks"] } sha2 = "0.10.9" +smallvec = "1.15.1" socket2 = "0.6.3" strum = { version = "0.27.2", features = ["derive"] } subenum = "1.1.3" diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs index f865e5a6ae..b511bb2424 100644 --- a/rust/src/chat/src/output/default/unified.rs +++ b/rust/src/chat/src/output/default/unified.rs @@ -13,7 +13,7 @@ use futures::{StreamExt as _, pin_mut}; use thiserror_ext::AsReport; use tracing::warn; use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; -use vllm_text::output::DecodedTextEvent; +use vllm_text::output::{DecodedTextEvent, SampledDelta}; use crate::Result; use crate::error::Error; @@ -237,12 +237,15 @@ pub(crate) async fn unified_event_stream( .await; } DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, + decoded, + sampled: + SampledDelta { + token_ids, + logprobs, + }, finished, } => { - for next in state.process_delta(delta)? { + for next in state.process_delta(decoded.text)? { y.yield_ok(next).await; } if logprobs.is_some() || !token_ids.is_empty() { @@ -375,24 +378,22 @@ mod tests { fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { vllm_text::output::DecodedTextEvent::TextDelta { - delta: delta.to_string(), - token_ids: Vec::new(), - logprobs: None, + decoded: vllm_text::DecodedText::unattributed(delta), + sampled: vllm_text::SampledDelta::default(), finished: None, } } fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { vllm_text::output::DecodedTextEvent::TextDelta { - delta: delta.to_string(), - token_ids: Vec::new(), - logprobs: None, - finished: Some(vllm_text::output::Finished { + decoded: vllm_text::DecodedText::unattributed(delta), + sampled: vllm_text::SampledDelta::default(), + finished: Some(Box::new(vllm_text::output::Finished { usage: vllm_llm::TokenUsage::default(), finish_reason: crate::FinishReason::Stop(None), kv_transfer_params: None, ec_transfer_params: None, - }), + })), } } diff --git a/rust/src/chat/src/output/harmony/mod.rs b/rust/src/chat/src/output/harmony/mod.rs index 14206751bb..82f476efbe 100644 --- a/rust/src/chat/src/output/harmony/mod.rs +++ b/rust/src/chat/src/output/harmony/mod.rs @@ -11,7 +11,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use futures::StreamExt as _; use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role}; use openai_harmony::{HarmonyEncoding, StreamableParser}; -use vllm_text::output::DecodedTextEvent; +use vllm_text::output::{DecodedTextEvent, SampledDelta}; use crate::Result as ChatResult; use crate::error::{Error, Result}; @@ -345,9 +345,12 @@ async fn harmony_assistant_event_stream( .await; } DecodedTextEvent::TextDelta { - delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here - token_ids, - logprobs, + decoded: _, // harmony takes raw token IDs as input, so we ignore decoded text here + sampled: + SampledDelta { + token_ids, + logprobs, + }, finished, } => { for event in state.process_token_ids(&token_ids)? { diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index e798f09269..4410bc8133 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use futures::executor::block_on; use futures::{TryStreamExt as _, stream}; use openai_harmony::chat::{Message, Role}; -use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished}; +use vllm_text::output::{ + DecodedLogprobs, DecodedPositionLogprobs, DecodedText, DecodedTextEvent, Finished, SampledDelta, +}; use super::*; use crate::output::ChatOutputProcessor; @@ -59,6 +61,21 @@ fn finished() -> Finished { } } +fn decoded_tokens( + token_ids: Vec, + logprobs: Option, + finished: Option, +) -> DecodedTextEvent { + DecodedTextEvent::TextDelta { + decoded: DecodedText::default(), + sampled: SampledDelta { + token_ids, + logprobs, + }, + finished: finished.map(Box::new), + } +} + async fn collect_events( processor: HarmonyChatOutputProcessor, events: Vec, @@ -96,12 +113,7 @@ fn interrupted_final_message_is_preserved() { HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens[..tokens.len() - 1].to_vec(), - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())), ], )); @@ -135,12 +147,7 @@ fn eos_flush_preserves_trailing_replacement_text() { HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens, - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens, None, Some(finished())), ], )); @@ -157,12 +164,7 @@ fn interrupted_analysis_message_is_preserved() { HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens[..tokens.len() - 1].to_vec(), - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())), ], )); @@ -196,12 +198,7 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() { HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens, - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens, None, Some(finished())), ], )); @@ -225,12 +222,7 @@ fn multiple_messages_get_newline_separators() { HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens, - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens, None, Some(finished())), ], )); @@ -256,18 +248,8 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() { HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens[..midpoint].to_vec(), - logprobs: None, - finished: None, - }, - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens[midpoint..].to_vec(), - logprobs: None, - finished: Some(finished()), - }, + decoded_tokens(tokens[..midpoint].to_vec(), None, None), + decoded_tokens(tokens[midpoint..].to_vec(), None, Some(finished())), ], )); @@ -303,14 +285,13 @@ fn semantic_events_precede_same_update_logprobs() { HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(), vec![ decoded_start(), - DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: tokens, - logprobs: Some(DecodedLogprobs { + decoded_tokens( + tokens, + Some(DecodedLogprobs { positions: vec![DecodedPositionLogprobs { entries: vec![] }], }), - finished: Some(finished()), - }, + Some(finished()), + ), ], )); diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index e9bc14a5e4..19aec82e45 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -784,15 +784,14 @@ fn decoded_completion_stream( if chunks.is_empty() { events.push({ DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: Vec::new(), - logprobs: None, - finished: Some(Finished { + decoded: vllm_text::DecodedText::default(), + sampled: vllm_text::SampledDelta::default(), + finished: Some(Box::new(Finished { usage: Default::default(), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, - }), + })), } }); } else { @@ -805,10 +804,12 @@ fn decoded_completion_stream( ec_transfer_params: None, }); events.push(DecodedTextEvent::TextDelta { - delta: chunk.delta, - token_ids: chunk.token_ids, - logprobs: None, - finished, + decoded: vllm_text::DecodedText::unattributed(chunk.delta), + sampled: vllm_text::SampledDelta { + token_ids: chunk.token_ids, + logprobs: None, + }, + finished: finished.map(Box::new), }); } } @@ -864,7 +865,7 @@ fn incremental_decode_chunks( for token_id in chunk_token_ids.iter().copied() { decoder.push_token(token_id)?; while let Some(chunk) = decoder.next_chunk() { - delta.push_str(&chunk); + delta.push_str(&chunk.text); } } chunks.push(DecodedCompletionChunk { @@ -876,10 +877,10 @@ fn incremental_decode_chunks( let (last_chunk, _) = decoder.flush(None)?; if let Some(last_chunk) = last_chunk { if let Some(delta) = chunks.last_mut() { - delta.delta.push_str(&last_chunk); + delta.delta.push_str(&last_chunk.text); } else { chunks.push(DecodedCompletionChunk { - delta: last_chunk, + delta: last_chunk.text, token_ids: Vec::new(), }); } diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index d5403f03c7..c3f590af99 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -14,7 +14,7 @@ use tracing::{Span, info, info_span, warn}; use tracing_futures::Instrument as _; use uuid::Uuid; use vllm_llm::current_unix_timestamp_secs; -use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest}; +use vllm_text::{DecodedTextEvent, Prompt, SampledDelta, TextOutputStreamExt as _, TextRequest}; use super::convert::{self, ResponseOpts}; use super::{InferenceServer, pb}; @@ -264,17 +264,20 @@ impl pb::inference_server::Inference for InferenceServiceImpl { }) } Ok(DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, + decoded, + sampled: + SampledDelta { + token_ids, + logprobs, + }, finished, }) => Ok(pb::GenerateResponse { prompt_info: None, outputs: Some(convert::to_sequence_output( - &delta, + &decoded.text, &token_ids, logprobs.as_ref(), - finished.as_ref(), + finished.as_deref(), &response_opts, )), }), diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index ecb355299b..a122afbcd9 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -23,7 +23,7 @@ use tracing_futures::Instrument as _; use vllm_engine_core_client::protocol::output::StopReason; use vllm_text::tokenizer::Tokenizer; use vllm_text::{ - DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream, + DecodedPromptLogprobs, DecodedTextEvent, FinishReason, SampledDelta, TextOutputStream, TextOutputStreamExt as _, TextRequest, }; @@ -317,11 +317,15 @@ async fn completion_chunk_stream( } } Ok(DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, + decoded, + sampled: + SampledDelta { + token_ids, + logprobs, + }, finished, }) => { + let delta = decoded.text; // Prompt-only streaming already emitted the echoed prompt in the Start chunk. // The one generated token is only used to drive the engine to a finished event, // so hide its delta and forward only the terminal finish/usage metadata. @@ -561,8 +565,8 @@ mod tests { use itertools::Itertools as _; use vllm_engine_core_client::protocol::output::StopReason; use vllm_text::{ - DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent, - DecodedTokenLogprob, FinishReason, Finished, + DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedText, + DecodedTextEvent, DecodedTokenLogprob, FinishReason, Finished, SampledDelta, }; use super::{ @@ -570,6 +574,22 @@ mod tests { StreamResponseEnvelope, completion_chunk_stream, final_chunk, }; + fn decoded_delta( + text: impl Into, + token_ids: Vec, + logprobs: Option, + finished: Option, + ) -> DecodedTextEvent { + DecodedTextEvent::TextDelta { + decoded: DecodedText::unattributed(text), + sampled: SampledDelta { + token_ids, + logprobs, + }, + finished: finished.map(Box::new), + } + } + fn stream_envelope() -> Arc { Arc::new(StreamResponseEnvelope::new( "cmpl-1".to_string(), @@ -619,10 +639,10 @@ mod tests { prompt_token_ids: vec![1, 2, 3, 4, 5].into(), prompt_logprobs: None, }), - Ok(DecodedTextEvent::TextDelta { - delta: "h".to_string(), - token_ids: vec![b'h' as u32], - logprobs: Some(DecodedLogprobs { + Ok(decoded_delta( + "h", + vec![b'h' as u32], + Some(DecodedLogprobs { positions: vec![DecodedPositionLogprobs { entries: vec![ DecodedTokenLogprob { @@ -640,12 +660,12 @@ mod tests { ], }], }), - finished: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: String::new(), - token_ids: vec![b'!' as u32], - logprobs: Some(DecodedLogprobs { + None, + )), + Ok(decoded_delta( + "", + vec![b'!' as u32], + Some(DecodedLogprobs { positions: vec![DecodedPositionLogprobs { entries: vec![ DecodedTokenLogprob { @@ -663,7 +683,7 @@ mod tests { ], }], }), - finished: Some(Finished { + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 5, output_token_count: 2, @@ -675,7 +695,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let chunks = completion_chunk_stream( @@ -773,11 +793,11 @@ mod tests { prompt_token_ids: vec![1, 2].into(), prompt_logprobs: None, }), - Ok(DecodedTextEvent::TextDelta { - delta: " leaked".to_string(), - token_ids: vec![3], - logprobs: None, - finished: Some(Finished { + Ok(decoded_delta( + " leaked", + vec![3], + None, + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 1, @@ -787,7 +807,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let response = super::collect_completion( @@ -825,11 +845,11 @@ mod tests { prompt_token_ids: vec![9707].into(), prompt_logprobs: None, }), - Ok(DecodedTextEvent::TextDelta { - delta: " leaked".to_string(), - token_ids: vec![3], - logprobs: None, - finished: Some(Finished { + Ok(decoded_delta( + " leaked", + vec![3], + None, + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, @@ -839,7 +859,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let response = super::collect_completion( @@ -880,11 +900,11 @@ mod tests { prompt_token_ids: vec![1, 2].into(), prompt_logprobs: None, }), - Ok(DecodedTextEvent::TextDelta { - delta: " leaked".to_string(), - token_ids: vec![3], - logprobs: None, - finished: Some(Finished { + Ok(decoded_delta( + " leaked", + vec![3], + None, + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 1, @@ -894,7 +914,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let chunks = completion_chunk_stream( @@ -952,11 +972,11 @@ mod tests { prompt_token_ids: vec![9707].into(), prompt_logprobs: None, }), - Ok(DecodedTextEvent::TextDelta { - delta: " leaked".to_string(), - token_ids: vec![3], - logprobs: None, - finished: Some(Finished { + Ok(decoded_delta( + " leaked", + vec![3], + None, + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, @@ -966,7 +986,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let chunks = completion_chunk_stream( @@ -1026,11 +1046,11 @@ mod tests { }], }), }), - Ok(DecodedTextEvent::TextDelta { - delta: " leaked".to_string(), - token_ids: vec![3], - logprobs: None, - finished: Some(Finished { + Ok(decoded_delta( + " leaked", + vec![3], + None, + Some(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 1, @@ -1040,7 +1060,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, }), - }), + )), ]); let chunks = completion_chunk_stream( diff --git a/rust/src/text/src/lib.rs b/rust/src/text/src/lib.rs index 8357346b6f..b76c2263f2 100644 --- a/rust/src/text/src/lib.rs +++ b/rust/src/text/src/lib.rs @@ -19,7 +19,8 @@ pub use lower::{ }; pub use output::{ CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, - DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt, + DecodedText, DecodedTextEvent, DecodedTokenLogprob, Finished, SampledDelta, TextDecodeOptions, + TextOutputStreamExt, TokenAnchor, TokenAttribution, }; pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k}; use trait_set::trait_set; diff --git a/rust/src/text/src/output/decoded.rs b/rust/src/text/src/output/decoded.rs index a660e00581..0dec802f5b 100644 --- a/rust/src/text/src/output/decoded.rs +++ b/rust/src/text/src/output/decoded.rs @@ -10,7 +10,7 @@ use tracing::{Level, debug, trace}; use vllm_engine_core_client::AbortCause; use vllm_engine_core_client::protocol::output::StopReason; use vllm_llm::{FinishReason, GenerateOutput, TokenUsage}; -use vllm_tokenizer::{DynTokenizer, IncrementalDecoder}; +use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder}; use super::logprobs::{ DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs, @@ -52,6 +52,19 @@ pub struct Finished { pub ec_transfer_params: Option, } +/// Sample metadata emitted by one engine output update. +/// +/// Token IDs and logprobs share the engine-update clock and align by generated +/// token position. They may arrive before the corresponding decoded text is +/// visible or alongside decoded text attributed to earlier updates. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct SampledDelta { + /// Token IDs produced by this engine update, after stop-string truncation. + pub token_ids: Vec, + /// Per-position logprobs aligned with `token_ids`, when requested. + pub logprobs: Option, +} + /// Internal decoded-text event emitted before higher-level assistant /// adaptation. #[derive(Debug, Clone, PartialEq)] @@ -68,25 +81,25 @@ pub enum DecodedTextEvent { /// remaining prompt positions. prompt_logprobs: Option, }, - /// A delta of text has been decoded, optionally alongside token-position - /// logprobs. + /// Decoded text and sampled metadata observed during one output update. /// - /// `delta` is the newly visible decoded text fragment for this update. + /// `decoded` follows the decoder clock. It may be empty while tokens remain + /// undecodable, and may later contain token attributions from earlier + /// engine updates. Tokens suppressed before decoding are absent. /// - /// `logprobs` covers the newly generated token positions from the same - /// update, but is not guaranteed to align with `delta` by character - /// span. One update may carry token logprobs but no newly visible text - /// yet, and one visible text fragment may reflect multiple token - /// positions becoming decodable together. + /// `sampled` follows the engine-update clock. Its token IDs and logprobs + /// align with each other. `decoded.attributions` has an independent decoder + /// clock and positional sequence. /// - /// Upper-level may further parse `delta` as reasoning or tool calls. + /// Upper-level may further parse `decoded.text` as reasoning or tool calls. /// /// When `finished` is `Some`, this is the terminal event for the request. TextDelta { - delta: String, - token_ids: Vec, - logprobs: Option, - finished: Option, + decoded: DecodedText, + sampled: SampledDelta, + /// Heap-backed because terminal metadata is large and appears once per + /// request, while every streaming event carries this enum variant. + finished: Option>, }, } @@ -104,9 +117,9 @@ pub async fn decoded_text_event_stream( let mut decoder: Option> = None; let mut prompt_token_count = 0_usize; let mut cached_token_count = 0_usize; - let mut token_ids = Vec::new(); + let mut sampled_token_ids = Vec::new(); let mut output_token_count: usize = 0; - let mut logprobs: Option = None; + let mut sampled_logprobs: Option = None; while let Some(next) = raw_stream.next().await { let output = next?; @@ -168,7 +181,7 @@ pub async fn decoded_text_event_stream( &output.token_ids }; - let mut delta: Option = None; + let mut decoded = DecodedText::default(); let mut truncate_output_to = None; let mut truncate_tokens_to = None; for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() { @@ -189,12 +202,9 @@ pub async fn decoded_text_event_stream( break; } + // TODO: avoid generating a chunk every time a token is pushed if intermediate && let Some(chunk) = decoder.next_chunk() { - if let Some(delta_str) = delta.as_mut() { - delta_str.push_str(&chunk); - } else { - delta = Some(chunk); - } + decoded.append(chunk); } } @@ -223,9 +233,9 @@ pub async fn decoded_text_event_stream( .transpose()?; if !intermediate { - token_ids.extend(&new_token_ids); + sampled_token_ids.extend(&new_token_ids); if let Some(dlp) = decoded_logprobs.as_ref() { - logprobs + sampled_logprobs .get_or_insert_with(|| DecodedLogprobs { positions: vec![] }) .positions .extend_from_slice(&dlp.positions); @@ -234,22 +244,19 @@ pub async fn decoded_text_event_stream( if let Some(reason) = finish_reason { // Flush any remaining buffered text. - let (last_chunk, mut text) = decoder.flush(truncate_output_to)?; - let text_len = text.len(); - let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone()); - - if intermediate { + let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?; + let text_len = full_decoded.text.len(); + let full_text = tracing::enabled!(Level::TRACE).then(|| full_decoded.text.clone()); + let decoded = if intermediate { if let Some(chunk) = last_chunk { - if let Some(delta_str) = delta.as_mut() { - delta_str.push_str(&chunk); - } else { - delta = Some(chunk); - } + decoded.append(chunk); } - token_ids = new_token_ids; - logprobs = decoded_logprobs; - text = delta.unwrap_or_default(); - } + sampled_token_ids = new_token_ids; + sampled_logprobs = decoded_logprobs; + decoded + } else { + full_decoded + }; debug!( finish_reason = ?reason, @@ -269,10 +276,12 @@ pub async fn decoded_text_event_stream( } y.yield_ok(DecodedTextEvent::TextDelta { - delta: text, - token_ids, - logprobs, - finished: Some(Finished { + decoded, + sampled: SampledDelta { + token_ids: sampled_token_ids, + logprobs: sampled_logprobs, + }, + finished: Some(Box::new(Finished { usage: TokenUsage { prompt_token_count, output_token_count, @@ -281,7 +290,7 @@ pub async fn decoded_text_event_stream( finish_reason: reason, kv_transfer_params, ec_transfer_params, - }), + })), }) .await; return Ok(()); @@ -289,9 +298,11 @@ pub async fn decoded_text_event_stream( if intermediate { y.yield_ok(DecodedTextEvent::TextDelta { - delta: delta.unwrap_or_default(), - token_ids: new_token_ids, - logprobs: decoded_logprobs, + decoded, + sampled: SampledDelta { + token_ids: new_token_ids, + logprobs: decoded_logprobs, + }, finished: None, }) .await; @@ -338,10 +349,10 @@ mod tests { use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; - use futures::{Stream, stream}; + use futures::{Stream, StreamExt as _, stream}; use vllm_engine_core_client::AbortCause; use vllm_llm::GenerateOutput; - use vllm_tokenizer::test_utils::TestTokenizer; + use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer}; use super::*; use crate::output::TextOutputStreamExt as _; @@ -378,6 +389,129 @@ mod tests { } } + fn visible(token_id: u32, byte_offset: u32) -> TokenAttribution { + TokenAttribution { + token_id, + anchor: TokenAnchor::Visible { byte_offset }, + } + } + + fn text_delta( + text: impl Into, + attributions: impl IntoIterator, + sampled_token_ids: Vec, + finished: Option, + ) -> DecodedTextEvent { + DecodedTextEvent::TextDelta { + decoded: DecodedText { + text: text.into(), + attributions: attributions.into_iter().collect(), + }, + sampled: SampledDelta { + token_ids: sampled_token_ids, + logprobs: None, + }, + finished: finished.map(Box::new), + } + } + + async fn run_intermediate( + outputs: Vec, + decode_options: TextDecodeOptions, + ) -> Vec { + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + decoded_text_event_stream( + "test".into(), + tokenizer, + stream::iter(outputs.into_iter().map(Ok)), + decode_options, + true, + ) + .collect::>() + .await + .into_iter() + .collect::>>() + .unwrap() + } + + #[tokio::test] + async fn decoded_and_sampled_clocks_are_independent() { + struct Case { + name: &'static str, + outputs: Vec, + expected: Vec, + } + + let prompt: Arc<[u32]> = Arc::from([]); + let cases = [ + Case { + name: "utf8 byte fallback resolves across engine updates", + outputs: vec![ + GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None), + GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)), + ], + expected: vec![ + DecodedTextEvent::Start { + prompt_token_ids: Arc::clone(&prompt), + prompt_logprobs: None, + }, + text_delta("", [], vec![0xe4], None), + text_delta( + "你", + [visible(0xe4, 0), visible(0xbd, 0), visible(0xa0, 0)], + vec![0xbd, 0xa0], + Some(Finished { + usage: TokenUsage { + prompt_token_count: 0, + output_token_count: 3, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + ec_transfer_params: None, + }), + ), + ], + }, + Case { + name: "terminal stop token remains sampled and bypasses decoder", + outputs: vec![GenerateOutput::for_test( + Some(Arc::clone(&prompt)), + ascii_tokens("a!"), + Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))), + )], + expected: vec![ + DecodedTextEvent::Start { + prompt_token_ids: Arc::clone(&prompt), + prompt_logprobs: None, + }, + text_delta( + "a", + [visible(b'a' as u32, 0)], + ascii_tokens("a!"), + Some(Finished { + usage: TokenUsage { + prompt_token_count: 0, + output_token_count: 2, + cached_token_count: 0, + }, + finish_reason: FinishReason::Stop(Some(StopReason::TokenId( + b'!' as u32, + ))), + kv_transfer_params: None, + ec_transfer_params: None, + }), + ), + ], + }, + ]; + + for case in cases { + let actual = run_intermediate(case.outputs, TextDecodeOptions::default()).await; + assert_eq!(actual, case.expected, "{}", case.name); + } + } + struct DropRecordingStream { next: Option>, dropped_cause: Arc>>, diff --git a/rust/src/text/src/output/mod.rs b/rust/src/text/src/output/mod.rs index c3499a9f35..a415bb8900 100644 --- a/rust/src/text/src/output/mod.rs +++ b/rust/src/text/src/output/mod.rs @@ -3,10 +3,13 @@ //! Output processing helpers shared by text and chat layers. -pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream}; +pub use decoded::{ + DecodedTextEvent, Finished, SampledDelta, TextDecodeOptions, decoded_text_event_stream, +}; pub use logprobs::{ DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, }; +pub use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution}; mod decoded; mod logprobs; @@ -58,13 +61,16 @@ impl T { prompt_token_ids = start_prompt_token_ids; } DecodedTextEvent::TextDelta { - delta, - token_ids: delta_token_ids, - logprobs: mut delta_logprobs, + decoded, + sampled: + SampledDelta { + token_ids: delta_token_ids, + logprobs: mut delta_logprobs, + }, finished, } => { if let Some(c) = collected.as_mut() { - c.text.push_str(&delta); + c.text.push_str(&decoded.text); c.token_ids.extend(delta_token_ids); if let Some(dlp) = delta_logprobs.as_mut() { if let Some(lp) = c.logprobs.as_mut() { @@ -75,7 +81,7 @@ impl T { } } else { collected = Some(CollectedTextOutput { - text: delta, + text: decoded.text, prompt_token_ids: Arc::clone(&prompt_token_ids), prompt_logprobs: prompt_logprobs.take(), logprobs: delta_logprobs, @@ -134,29 +140,31 @@ mod tests { }), }), Ok(DecodedTextEvent::TextDelta { - delta: "bc".to_string(), - token_ids: vec![1, 2], - logprobs: Some(DecodedLogprobs { - positions: vec![ - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "a".to_string(), - logprob: -0.2, - rank: 1, - }], - }, - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "bc".to_string(), - logprob: -0.3, - rank: 1, - }], - }, - ], - }), - finished: Some(Finished { + decoded: DecodedText::unattributed("bc"), + sampled: SampledDelta { + token_ids: vec![1, 2], + logprobs: Some(DecodedLogprobs { + positions: vec![ + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "a".to_string(), + logprob: -0.2, + rank: 1, + }], + }, + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "bc".to_string(), + logprob: -0.3, + rank: 1, + }], + }, + ], + }), + }, + finished: Some(Box::new(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 2, @@ -165,7 +173,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, - }), + })), }), ]); @@ -219,62 +227,66 @@ mod tests { prompt_logprobs: None, }), Ok(DecodedTextEvent::TextDelta { - delta: "he".to_string(), - token_ids: vec![1, 2], - logprobs: Some(DecodedLogprobs { - positions: vec![ - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "h".to_string(), - logprob: -0.1, - rank: 1, - }], - }, - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "e".to_string(), - logprob: -0.2, - rank: 1, - }], - }, - ], - }), + decoded: DecodedText::unattributed("he"), + sampled: SampledDelta { + token_ids: vec![1, 2], + logprobs: Some(DecodedLogprobs { + positions: vec![ + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "h".to_string(), + logprob: -0.1, + rank: 1, + }], + }, + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "e".to_string(), + logprob: -0.2, + rank: 1, + }], + }, + ], + }), + }, finished: None, }), Ok(DecodedTextEvent::TextDelta { - delta: "llo".to_string(), - token_ids: vec![3, 4, 5], - logprobs: Some(DecodedLogprobs { - positions: vec![ - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "l".to_string(), - logprob: -0.3, - rank: 1, - }], - }, - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "l".to_string(), - logprob: -0.4, - rank: 1, - }], - }, - DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "o".to_string(), - logprob: -0.5, - rank: 1, - }], - }, - ], - }), - finished: Some(Finished { + decoded: DecodedText::unattributed("llo"), + sampled: SampledDelta { + token_ids: vec![3, 4, 5], + logprobs: Some(DecodedLogprobs { + positions: vec![ + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "l".to_string(), + logprob: -0.3, + rank: 1, + }], + }, + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "l".to_string(), + logprob: -0.4, + rank: 1, + }], + }, + DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 0, + token: "o".to_string(), + logprob: -0.5, + rank: 1, + }], + }, + ], + }), + }, + finished: Some(Box::new(Finished { usage: vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 5, @@ -283,7 +295,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, - }), + })), }), ]); diff --git a/rust/src/tokenizer/Cargo.toml b/rust/src/tokenizer/Cargo.toml index d18b66b231..bf35c469b8 100644 --- a/rust/src/tokenizer/Cargo.toml +++ b/rust/src/tokenizer/Cargo.toml @@ -14,6 +14,7 @@ riptoken.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +smallvec.workspace = true tekken.workspace = true thiserror.workspace = true thiserror-ext.workspace = true diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index f608fa874b..3c9c164798 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -1,10 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -use std::mem::take; - use crate::{Result, Tokenizer}; +mod attribution; + +use attribution::AttributedTextBuffer; +pub use attribution::{DecodedText, TokenAnchor, TokenAttribution}; + /// Stateful incremental decoder that emits text chunks one token at a time. pub trait IncrementalDecoder: Send { /// Push one generated token and return how many new string bytes were @@ -12,13 +15,16 @@ pub trait IncrementalDecoder: Send { fn push_token(&mut self, token_id: u32) -> Result; /// Consume any text which is currently ready. - fn next_chunk(&mut self) -> Option; + fn next_chunk(&mut self) -> Option; /// Flush any remaining buffered text that has not yet been emitted. /// /// Called after the final generated token to force out buffered/incomplete /// fragments. - fn flush(&mut self, truncate_output_to: Option) -> Result<(Option, String)>; + fn flush( + &mut self, + truncate_output_to: Option, + ) -> Result<(Option, DecodedText)>; /// Return cumulative decoded text so far. fn output(&self) -> &str; @@ -36,8 +42,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> { prefix: String, prefix_index: usize, prefix_seeded: bool, - cumulative_output: String, - output_index: usize, + decoded: AttributedTextBuffer, } impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> { @@ -55,8 +60,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> { prefix: String::new(), prefix_index: 0, prefix_seeded: prompt_token_ids.is_empty(), - cumulative_output: String::new(), - output_index: 0, + decoded: AttributedTextBuffer::default(), } } } @@ -68,6 +72,14 @@ const SAFE_SUFFIX_MIN: usize = 4; const SAFE_SUFFIX_MAX: usize = 6; impl DecodeStream<'_, T> { + fn record_token(&mut self, token_id: u32) { + if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) { + self.decoded.record_zero_width_token(token_id); + } else { + self.decoded.record_pending_token(token_id); + } + } + /// Decode prompt-only context for prefix seeding. /// /// Prompt ids may come from the model vocabulary rather than the local @@ -132,6 +144,7 @@ impl IncrementalDecoder for DecodeStream<'_, T> { } self.ids.push(token_id); + self.record_token(token_id); let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?; let prefix_len = self.prefix.len(); if string.len() <= prefix_len || string.ends_with('\u{FFFD}') { @@ -139,49 +152,48 @@ impl IncrementalDecoder for DecodeStream<'_, T> { } // Ensure we split at a utf-8 char boundary. let new_chunk = &string[string.floor_char_boundary(prefix_len)..]; - self.cumulative_output.push_str(new_chunk); + self.decoded.append_visible_text(new_chunk); self.ids.drain(..self.prefix_index); self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?; self.prefix_index = self.ids.len(); Ok(new_chunk.len()) } - fn next_chunk(&mut self) -> Option { - let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer); + fn next_chunk(&mut self) -> Option { + let cutoff = self.decoded.len().saturating_sub(self.min_bytes_to_buffer); // Ensure we split at a utf-8 char boundary. - let cutoff = self.cumulative_output.floor_char_boundary(cutoff); - (cutoff > self.output_index).then(|| { - let chunk = self.cumulative_output[self.output_index..cutoff].to_string(); - self.output_index = cutoff; - chunk - }) + let cutoff = self.decoded.text().floor_char_boundary(cutoff); + self.decoded.take_ready(cutoff) } - fn flush(&mut self, truncate_output_to: Option) -> Result<(Option, String)> { + fn flush( + &mut self, + truncate_output_to: Option, + ) -> Result<(Option, DecodedText)> { // If the prefix was never seeded (no push_token was called), `ids` // holds only prompt context — decoding it would re-emit prompt text. if self.prefix_seeded && !self.ids.is_empty() { let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?; let prefix_len = self.prefix.len(); // Ensure we split at a utf-8 char boundary. - self.cumulative_output - .push_str(&string[string.floor_char_boundary(prefix_len)..]); + let new_chunk = &string[string.floor_char_boundary(prefix_len)..]; + if !new_chunk.is_empty() { + self.decoded.append_visible_text(new_chunk); + } } + self.decoded.resolve_pending_zero_width(); self.ids.clear(); self.prefix.clear(); self.prefix_index = 0; self.prefix_seeded = true; if let Some(truncate_output_to) = truncate_output_to { - self.cumulative_output.truncate(truncate_output_to); + self.decoded.truncate(truncate_output_to); } - let last_chunk = (self.output_index < self.cumulative_output.len()) - .then(|| self.cumulative_output[self.output_index..].to_string()); - self.output_index = 0; - Ok((last_chunk, take(&mut self.cumulative_output))) + Ok(self.decoded.finish()) } fn output(&self) -> &str { - &self.cumulative_output + self.decoded.text() } } @@ -245,13 +257,13 @@ mod tests { let mut decoder = backend.create_decode_stream(&[], false, 0); assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1); - assert_eq!(decoder.next_chunk().as_deref(), Some("o")); + assert_eq!(decoder.next_chunk().unwrap().text, "o"); assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1); - assert_eq!(decoder.next_chunk().as_deref(), Some("k")); + assert_eq!(decoder.next_chunk().unwrap().text, "k"); // All text already consumed via next_chunk let (last_chunk, full_text) = decoder.flush(None).unwrap(); assert_eq!(last_chunk, None); - assert_eq!(full_text, "ok"); + assert_eq!(full_text.text, "ok"); } #[test] @@ -288,6 +300,7 @@ mod tests { 0 if !skip_special_tokens => text.push_str(""), 0 => {} 1 => text.push('a'), + 2 => text.push('b'), _ => {} } } @@ -301,6 +314,10 @@ mod tests { fn id_to_token(&self, _id: u32) -> Option { unreachable!() } + + fn is_special_id(&self, token_id: u32) -> bool { + token_id == 0 + } } #[test] @@ -393,17 +410,17 @@ mod tests { let mut decoder = backend.create_decode_stream(&[], false, 0); let input = b"Hello, world!"; - let mut full = String::new(); + let mut out = String::new(); for &byte in input { decoder.push_token(byte as u32).unwrap(); if let Some(chunk) = decoder.next_chunk() { - full.push_str(&chunk); + out.push_str(&chunk.text); } } - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); assert_eq!(last_chunk, None); // all consumed via next_chunk - assert_eq!(full, "Hello, world!"); - assert_eq!(full_text, "Hello, world!"); + assert_eq!(out, "Hello, world!"); + assert_eq!(full.text, "Hello, world!"); } /// Backend simulating non-monotonic decode where adding a token changes how @@ -470,15 +487,15 @@ mod tests { for &byte in input { decoder.push_token(byte as u32).unwrap(); if let Some(chunk) = decoder.next_chunk() { - chunks.push_str(&chunk); + chunks.push_str(&chunk.text); } } // With hold_back_bytes=3, last 3 bytes ("lo!") are held back assert_eq!(chunks, "Hel"); // Flush returns the rest - let (last_chunk, full_text) = decoder.flush(None).unwrap(); - assert_eq!(last_chunk.as_deref(), Some("lo!")); - assert_eq!(full_text, "Hello!"); + let (last_chunk, full) = decoder.flush(None).unwrap(); + assert_eq!(last_chunk.unwrap().text, "lo!"); + assert_eq!(full.text, "Hello!"); } #[test] @@ -493,14 +510,14 @@ mod tests { for byte in "你好A".bytes() { decoder.push_token(u32::from(byte)).unwrap(); if let Some(chunk) = decoder.next_chunk() { - out.push_str(&chunk); + out.push_str(&chunk.text); } } - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); if let Some(chunk) = last_chunk { - out.push_str(&chunk); + out.push_str(&chunk.text); } - assert_eq!(full_text, "你好A"); + assert_eq!(full.text, "你好A"); assert_eq!(out, "你好A"); } @@ -515,9 +532,9 @@ mod tests { .collect(); let mut decoder = backend.create_decode_stream(&prompt, false, 0); - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); assert_eq!(last_chunk, None); - assert_eq!(full_text, ""); + assert_eq!(full.text, ""); } #[test] @@ -526,8 +543,8 @@ mod tests { let prompt = vec![0xe4, 0xbd]; let mut decoder = backend.create_decode_stream(&prompt, false, 0); - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); assert_eq!(last_chunk, None); - assert_eq!(full_text, ""); + assert_eq!(full.text, ""); } } diff --git a/rust/src/tokenizer/src/incremental/attribution.rs b/rust/src/tokenizer/src/incremental/attribution.rs new file mode 100644 index 0000000000..358d83f421 --- /dev/null +++ b/rust/src/tokenizer/src/incremental/attribution.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::mem::take; + +use smallvec::SmallVec; + +/// Position of one generated token in decoded text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TokenAnchor { + /// The token contributed to visible decoded text. + Visible { byte_offset: u32 }, + /// The token produced no visible decoded bytes. + ZeroWidth { byte_offset: u32 }, +} + +impl TokenAnchor { + fn byte_offset(self) -> u32 { + match self { + Self::Visible { byte_offset } | Self::ZeroWidth { byte_offset } => byte_offset, + } + } + + /// Return a new anchor offset by `delta` bytes. + #[must_use] + pub(super) fn offset_by(self, delta: i64) -> Self { + let byte_offset = i64::from(self.byte_offset()) + .checked_add(delta) + .and_then(|offset| u32::try_from(offset).ok()) + .expect("token anchor byte offset out of range"); + match self { + Self::Visible { .. } => Self::Visible { byte_offset }, + Self::ZeroWidth { .. } => Self::ZeroWidth { byte_offset }, + } + } +} + +/// One generated token and its position in decoded text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TokenAttribution { + /// Token ID passed to [`super::IncrementalDecoder::push_token`]. + pub token_id: u32, + /// Position of the token in the decoded text. + pub anchor: TokenAnchor, +} + +/// Decoded text and the generated tokens attributed to it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DecodedText { + /// Decoded UTF-8 text. + pub text: String, + /// One entry per generated token, in generation order. + /// + /// Offsets are local to `text`. Visible offsets may repeat when multiple + /// tokens jointly decode into one character. A zero-width offset may equal + /// `text.len()`. + /// Four inline records cover one complete UTF-8 byte-fallback sequence. + pub attributions: SmallVec<[TokenAttribution; 4]>, +} + +impl DecodedText { + /// Create decoded text without token attribution records. Mostly for testing purposes. + pub fn unattributed(text: impl Into) -> Self { + Self { + text: text.into(), + attributions: SmallVec::new(), + } + } + + /// Append another decoded fragment, rebasing its token anchors. + pub fn append(&mut self, other: Self) { + if self.text.is_empty() && self.attributions.is_empty() { + *self = other; + return; + } + + let byte_offset = offset_as_u32(self.text.len()); + let other_len = offset_as_u32(other.text.len()); + let _combined_len = byte_offset.checked_add(other_len).expect("decoded text exceeds 4 GiB"); + + self.text.push_str(&other.text); + self.attributions.extend(other.attributions.into_iter().map(|attribution| { + TokenAttribution { + token_id: attribution.token_id, + anchor: attribution.anchor.offset_by(i64::from(byte_offset)), + } + })); + } +} + +#[derive(Clone, Copy, Debug)] +enum PendingAnchor { + Unresolved, + Resolved(TokenAnchor), +} + +#[derive(Clone, Copy, Debug)] +struct PendingAttribution { + token_id: u32, + anchor: PendingAnchor, +} + +/// Cumulative decoded text plus the attribution state needed to emit local +/// chunks. +#[derive(Default)] +pub(super) struct AttributedTextBuffer { + text: String, + emitted_byte_offset: usize, + attributions: Vec, + emitted_attribution_count: usize, + pending_start: Option, +} + +impl AttributedTextBuffer { + /// Return all decoded text accumulated so far. + pub(super) fn text(&self) -> &str { + &self.text + } + + /// Return the decoded text length in bytes. + pub(super) fn len(&self) -> usize { + self.text.len() + } + + /// Record a token awaiting visible decoded output. + pub(super) fn record_pending_token(&mut self, token_id: u32) { + self.pending_start.get_or_insert(self.attributions.len()); + self.attributions.push(PendingAttribution { + token_id, + anchor: PendingAnchor::Unresolved, + }); + } + + /// Record a token that produced no visible decoded output. + pub(super) fn record_zero_width_token(&mut self, token_id: u32) { + self.attributions.push(PendingAttribution { + token_id, + anchor: PendingAnchor::Resolved(TokenAnchor::ZeroWidth { + byte_offset: offset_as_u32(self.text.len()), + }), + }); + } + + /// Append visible text and anchor pending tokens at its start. + pub(super) fn append_visible_text(&mut self, text: &str) { + let byte_offset = offset_as_u32(self.text.len()); + if let Some(pending_start) = self.pending_start.take() { + for attribution in &mut self.attributions[pending_start..] { + if matches!(attribution.anchor, PendingAnchor::Unresolved) { + attribution.anchor = + PendingAnchor::Resolved(TokenAnchor::Visible { byte_offset }); + } + } + } + self.text.push_str(text); + } + + /// Resolve remaining pending tokens at the current text boundary. + pub(super) fn resolve_pending_zero_width(&mut self) { + let Some(pending_start) = self.pending_start.take() else { + return; + }; + let byte_offset = offset_as_u32(self.text.len()); + for attribution in &mut self.attributions[pending_start..] { + if matches!(attribution.anchor, PendingAnchor::Unresolved) { + attribution.anchor = + PendingAnchor::Resolved(TokenAnchor::ZeroWidth { byte_offset }); + } + } + } + + /// Truncate text and collapse removed anchors to the new boundary. + pub(super) fn truncate(&mut self, byte_offset: usize) { + self.text.truncate(byte_offset); + let byte_offset = offset_as_u32(byte_offset); + for attribution in &mut self.attributions { + let PendingAnchor::Resolved(anchor) = &mut attribution.anchor else { + continue; + }; + if anchor.byte_offset() >= byte_offset { + *anchor = TokenAnchor::ZeroWidth { byte_offset }; + } + } + } + + /// Emit decoded text and attributions ready through `cutoff`. + pub(super) fn take_ready(&mut self, cutoff: usize) -> Option { + let chunk_start = self.emitted_byte_offset; + let cutoff_u32 = offset_as_u32(cutoff); + let mut attribution_end = self.emitted_attribution_count; + + for attribution in &self.attributions[self.emitted_attribution_count..] { + let PendingAnchor::Resolved(anchor) = attribution.anchor else { + break; + }; + let ready = match anchor { + TokenAnchor::Visible { byte_offset } => byte_offset < cutoff_u32, + TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= cutoff_u32, + }; + if !ready { + break; + } + attribution_end += 1; + } + + if cutoff == chunk_start && attribution_end == self.emitted_attribution_count { + return None; + } + + let chunk_start_u32 = offset_as_u32(chunk_start); + let attributions = self.attributions[self.emitted_attribution_count..attribution_end] + .iter() + .map(|attribution| { + let PendingAnchor::Resolved(anchor) = attribution.anchor else { + unreachable!("ready attributions must be resolved") + }; + TokenAttribution { + token_id: attribution.token_id, + anchor: anchor.offset_by(-i64::from(chunk_start_u32)), + } + }) + .collect(); + let text = self.text[chunk_start..cutoff].to_string(); + + self.emitted_byte_offset = cutoff; + self.emitted_attribution_count = attribution_end; + Some(DecodedText { text, attributions }) + } + + /// Emit the final chunk and take the complete decoded output. + pub(super) fn finish(&mut self) -> (Option, DecodedText) { + let last_chunk = self.take_ready(self.text.len()); + let attributions = take(&mut self.attributions) + .into_iter() + .map(|attribution| { + let PendingAnchor::Resolved(anchor) = attribution.anchor else { + unreachable!("finish requires every attribution to be resolved") + }; + TokenAttribution { + token_id: attribution.token_id, + anchor, + } + }) + .collect(); + let full = DecodedText { + text: take(&mut self.text), + attributions, + }; + self.emitted_byte_offset = 0; + self.emitted_attribution_count = 0; + self.pending_start = None; + (last_chunk, full) + } +} + +fn offset_as_u32(offset: usize) -> u32 { + u32::try_from(offset).expect("decoded text exceeds 4 GiB") +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/tokenizer/src/incremental/attribution/tests.rs b/rust/src/tokenizer/src/incremental/attribution/tests.rs new file mode 100644 index 0000000000..26b4acfefc --- /dev/null +++ b/rust/src/tokenizer/src/incremental/attribution/tests.rs @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use super::*; +use crate::{Result, Tokenizer, test_utils::TestTokenizer}; + +const SPECIAL_TOKEN_ID: u32 = 0x100; + +/// Backend with tokens that decode to whole text pieces. +#[derive(Debug)] +struct PieceBackend; + +impl Tokenizer for PieceBackend { + fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result> { + unreachable!() + } + + fn encode_ordinary(&self, _text: &str) -> Result> { + unreachable!() + } + + fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result { + Ok(token_ids + .iter() + .map(|token_id| match token_id { + 1 => "abcd", + 2 => "ab", + 3 => "", + 4 => "", + _ => unreachable!("unexpected token id: {token_id}"), + }) + .collect()) + } + + fn token_to_id(&self, _token: &str) -> Option { + unreachable!() + } + + fn id_to_token(&self, id: u32) -> Option { + Some(id.to_string()) + } +} + +/// Raw-byte backend with one filterable special token. +#[derive(Debug)] +struct Utf8SpecialBackend; + +impl Tokenizer for Utf8SpecialBackend { + fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result> { + unreachable!() + } + + fn encode_ordinary(&self, _text: &str) -> Result> { + unreachable!() + } + + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { + let mut bytes = Vec::new(); + for &token_id in token_ids { + if token_id == SPECIAL_TOKEN_ID { + if !skip_special_tokens { + bytes.extend_from_slice(b""); + } + } else { + bytes.push(token_id as u8); + } + } + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } + + fn token_to_id(&self, _token: &str) -> Option { + unreachable!() + } + + fn id_to_token(&self, id: u32) -> Option { + Some(id.to_string()) + } + + fn is_special_id(&self, token_id: u32) -> bool { + token_id == SPECIAL_TOKEN_ID + } +} + +fn tokenizer_with_special_token() -> TestTokenizer { + TestTokenizer::new().with_special_token("", SPECIAL_TOKEN_ID) +} + +fn visible(byte_offset: u32) -> TokenAnchor { + TokenAnchor::Visible { byte_offset } +} + +fn zero_width(byte_offset: u32) -> TokenAnchor { + TokenAnchor::ZeroWidth { byte_offset } +} + +fn decoded(text: &str, token_ids: &[u32], anchors: &[TokenAnchor]) -> DecodedText { + assert_eq!(token_ids.len(), anchors.len()); + DecodedText { + text: text.to_string(), + attributions: token_ids + .iter() + .copied() + .zip(anchors.iter().copied()) + .map(|(token_id, anchor)| TokenAttribution { token_id, anchor }) + .collect(), + } +} + +#[test] +fn token_anchor_offset_by_preserves_anchor_kind() { + for (anchor, delta, expected) in [ + (visible(3), -2, visible(1)), + (zero_width(1), 4, zero_width(5)), + ] { + assert_eq!(anchor.offset_by(delta), expected); + } +} + +#[test] +fn decoded_text_append_rebases_attributions() { + let mut combined = decoded("a", &[1], &[visible(0)]); + combined.append(decoded( + "你", + &[2, 3, 4, 5], + &[visible(0), visible(0), visible(0), zero_width(3)], + )); + + assert_eq!( + combined, + decoded( + "a你", + &[1, 2, 3, 4, 5], + &[ + visible(0), + visible(1), + visible(1), + visible(1), + zero_width(4), + ], + ) + ); +} + +#[test] +fn decoded_text_append_moves_allocations_into_empty_receiver() { + let source = decoded( + "hello", + &[1, 2, 3, 4, 5], + &[visible(0), visible(1), visible(2), visible(3), visible(4)], + ); + let text_ptr = source.text.as_ptr(); + let attributions_ptr = source.attributions.as_ptr(); + let mut combined = DecodedText::default(); + + combined.append(source); + + assert_eq!(combined.text.as_ptr(), text_ptr); + assert_eq!(combined.attributions.as_ptr(), attributions_ptr); +} + +#[test] +fn decoded_text_append_preserves_zero_width_only_receiver() { + let mut combined = decoded("", &[1], &[zero_width(0)]); + combined.append(decoded("a", &[2], &[visible(0)])); + + assert_eq!( + combined, + decoded("a", &[1, 2], &[zero_width(0), visible(0)]) + ); +} + +struct AttributionCase<'a> { + tokenizer: &'a dyn Tokenizer, + prompt_token_ids: Vec, + token_ids: Vec, + skip_special_tokens: bool, + min_bytes_to_buffer: usize, + truncate_output_to: Option, + expected_chunks: Vec, + expected_full: DecodedText, +} + +fn run_attribution_case(case: AttributionCase<'_>) { + let mut decoder = case.tokenizer.create_decode_stream( + &case.prompt_token_ids, + case.skip_special_tokens, + case.min_bytes_to_buffer, + ); + let mut chunks = Vec::new(); + + for token_id in &case.token_ids { + decoder.push_token(*token_id).unwrap(); + while let Some(chunk) = decoder.next_chunk() { + chunks.push(chunk); + } + } + let (remaining, full) = decoder.flush(case.truncate_output_to).unwrap(); + if let Some(remaining) = remaining { + chunks.push(remaining); + } + + assert_eq!(chunks, case.expected_chunks, "chunks"); + assert_eq!(full, case.expected_full, "full output"); + assert_eq!( + full.attributions.len(), + case.token_ids.len(), + "one attribution per pushed token" + ); + assert_eq!( + full.attributions + .iter() + .map(|attribution| attribution.token_id) + .collect::>(), + case.token_ids, + "attribution preserves pushed token IDs" + ); + + let mut reconstructed_text = String::new(); + let mut reconstructed_attributions = Vec::new(); + for chunk in &chunks { + let chunk_start = reconstructed_text.len() as u32; + reconstructed_text.push_str(&chunk.text); + reconstructed_attributions.extend(chunk.attributions.iter().map(|attribution| { + TokenAttribution { + token_id: attribution.token_id, + anchor: attribution.anchor.offset_by(i64::from(chunk_start)), + } + })); + } + assert_eq!( + reconstructed_text, full.text, + "chunk text reconstructs full text" + ); + assert_eq!( + reconstructed_attributions.as_slice(), + full.attributions.as_slice(), + "chunk attributions reconstruct full attributions" + ); +} + +#[test] +fn attribution_ascii_tokens_have_one_visible_anchor_each() { + let tokenizer = TestTokenizer::new(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![b'o' as u32, b'k' as u32], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![ + decoded("o", &[b'o' as u32], &[visible(0)]), + decoded("k", &[b'k' as u32], &[visible(0)]), + ], + expected_full: decoded("ok", &[b'o' as u32, b'k' as u32], &[visible(0), visible(1)]), + }); +} + +#[test] +fn attribution_byte_fallback_tokens_share_one_visible_anchor() { + let tokenizer = TestTokenizer::new(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, 0xbd, 0xa0], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded( + "你", + &[0xe4, 0xbd, 0xa0], + &[visible(0), visible(0), visible(0)], + )], + expected_full: decoded( + "你", + &[0xe4, 0xbd, 0xa0], + &[visible(0), visible(0), visible(0)], + ), + }); +} + +#[test] +fn attribution_holdback_splits_text_without_repeating_token() { + run_attribution_case(AttributionCase { + tokenizer: &PieceBackend, + prompt_token_ids: vec![], + token_ids: vec![1], + skip_special_tokens: false, + min_bytes_to_buffer: 2, + truncate_output_to: None, + expected_chunks: vec![decoded("ab", &[1], &[visible(0)]), decoded("cd", &[], &[])], + expected_full: decoded("abcd", &[1], &[visible(0)]), + }); +} + +#[test] +fn attribution_filtered_special_token_is_zero_width_at_its_byte_boundary() { + let tokenizer = tokenizer_with_special_token(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32], + skip_special_tokens: true, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![ + decoded("a", &[b'a' as u32], &[visible(0)]), + decoded("", &[SPECIAL_TOKEN_ID], &[zero_width(0)]), + decoded("b", &[b'b' as u32], &[visible(0)]), + ], + expected_full: decoded( + "ab", + &[b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32], + &[visible(0), zero_width(1), visible(1)], + ), + }); +} + +#[test] +fn attribution_zero_width_preserves_order_inside_byte_fallback_group() { + run_attribution_case(AttributionCase { + tokenizer: &Utf8SpecialBackend, + prompt_token_ids: vec![], + token_ids: vec![0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0], + skip_special_tokens: true, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded( + "你", + &[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0], + &[visible(0), zero_width(0), visible(0), visible(0)], + )], + expected_full: decoded( + "你", + &[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0], + &[visible(0), zero_width(0), visible(0), visible(0)], + ), + }); +} + +#[test] +fn attribution_retained_special_token_has_visible_anchor() { + run_attribution_case(AttributionCase { + tokenizer: &Utf8SpecialBackend, + prompt_token_ids: vec![], + token_ids: vec![SPECIAL_TOKEN_ID], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded("", &[SPECIAL_TOKEN_ID], &[visible(0)])], + expected_full: decoded("", &[SPECIAL_TOKEN_ID], &[visible(0)]), + }); +} + +#[test] +fn attribution_empty_decode_resolves_to_zero_width_on_flush() { + run_attribution_case(AttributionCase { + tokenizer: &PieceBackend, + prompt_token_ids: vec![], + token_ids: vec![4], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded("", &[4], &[zero_width(0)])], + expected_full: decoded("", &[4], &[zero_width(0)]), + }); +} + +#[test] +fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() { + let tokenizer = TestTokenizer::new(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, 0xbd], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)])], + expected_full: decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)]), + }); +} + +#[test] +fn attribution_truncation_converts_fully_removed_token_to_zero_width() { + run_attribution_case(AttributionCase { + tokenizer: &PieceBackend, + prompt_token_ids: vec![], + token_ids: vec![2, 3], + skip_special_tokens: false, + min_bytes_to_buffer: 32, + truncate_output_to: Some(2), + expected_chunks: vec![decoded("ab", &[2, 3], &[visible(0), zero_width(2)])], + expected_full: decoded("ab", &[2, 3], &[visible(0), zero_width(2)]), + }); +} + +#[test] +fn attribution_truncation_inside_token_retains_first_byte_anchor() { + run_attribution_case(AttributionCase { + tokenizer: &PieceBackend, + prompt_token_ids: vec![], + token_ids: vec![1], + skip_special_tokens: false, + min_bytes_to_buffer: 32, + truncate_output_to: Some(2), + expected_chunks: vec![decoded("ab", &[1], &[visible(0)])], + expected_full: decoded("ab", &[1], &[visible(0)]), + }); +} + +#[test] +fn attribution_excludes_prompt_context() { + let tokenizer = TestTokenizer::new(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![b'H' as u32, b'i' as u32], + token_ids: vec![b'!' as u32], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded("!", &[b'!' as u32], &[visible(0)])], + expected_full: decoded("!", &[b'!' as u32], &[visible(0)]), + }); +} diff --git a/rust/src/tokenizer/src/lib.rs b/rust/src/tokenizer/src/lib.rs index 2bc1cfca3d..6b366d7285 100644 --- a/rust/src/tokenizer/src/lib.rs +++ b/rust/src/tokenizer/src/lib.rs @@ -17,7 +17,7 @@ mod tiktoken; pub use error::{Result, TokenizerError}; pub use hf::HuggingFaceTokenizer; -pub use incremental::IncrementalDecoder; +pub use incremental::{DecodedText, IncrementalDecoder, TokenAnchor, TokenAttribution}; pub use tekken::TekkenTokenizer; pub use tiktoken::TiktokenTokenizer; diff --git a/rust/src/tokenizer/src/tiktoken.rs b/rust/src/tokenizer/src/tiktoken.rs index fb569d69c3..9d3754a177 100644 --- a/rust/src/tokenizer/src/tiktoken.rs +++ b/rust/src/tokenizer/src/tiktoken.rs @@ -964,16 +964,16 @@ mod tests { for &id in &ids { decoder.push_token(id).unwrap(); if let Some(chunk) = decoder.next_chunk() { - output.push_str(&chunk); + output.push_str(&chunk.text); } } - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); if let Some(chunk) = last_chunk { - output.push_str(&chunk); + output.push_str(&chunk.text); } assert_eq!(output, text); - assert_eq!(full_text, text); + assert_eq!(full.text, text); } } @@ -991,16 +991,16 @@ mod tests { for &id in &ids { decoder.push_token(id).unwrap(); if let Some(chunk) = decoder.next_chunk() { - output.push_str(&chunk); + output.push_str(&chunk.text); } } - let (last_chunk, full_text) = decoder.flush(None).unwrap(); + let (last_chunk, full) = decoder.flush(None).unwrap(); if let Some(chunk) = last_chunk { - output.push_str(&chunk); + output.push_str(&chunk.text); } assert_eq!(output, text); - assert_eq!(full_text, text); + assert_eq!(full.text, text); } }