[Rust Frontend] Attribute decoded text to tokens (#52910)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -13,7 +13,7 @@ use futures::{StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport;
|
||||
use tracing::warn;
|
||||
use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
use vllm_text::output::{DecodedTextEvent, SampledDelta};
|
||||
|
||||
use crate::Result;
|
||||
use crate::error::Error;
|
||||
@@ -237,12 +237,15 @@ pub(crate) async fn unified_event_stream(
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
for next in state.process_delta(delta)? {
|
||||
for next in state.process_delta(decoded.text)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
if logprobs.is_some() || !token_ids.is_empty() {
|
||||
@@ -375,24 +378,22 @@ mod tests {
|
||||
|
||||
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
delta: delta.to_string(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
delta: delta.to_string(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(vllm_text::output::Finished {
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: Some(Box::new(vllm_text::output::Finished {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::StreamExt as _;
|
||||
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
|
||||
use openai_harmony::{HarmonyEncoding, StreamableParser};
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
use vllm_text::output::{DecodedTextEvent, SampledDelta};
|
||||
|
||||
use crate::Result as ChatResult;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -345,9 +345,12 @@ async fn harmony_assistant_event_stream(
|
||||
.await;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: _, // harmony takes raw token IDs as input, so we ignore text deltas here
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded: _, // harmony takes raw token IDs as input, so we ignore decoded text here
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
for event in state.process_token_ids(&token_ids)? {
|
||||
|
||||
@@ -6,7 +6,9 @@ use std::sync::Arc;
|
||||
use futures::executor::block_on;
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use openai_harmony::chat::{Message, Role};
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
|
||||
use vllm_text::output::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedText, DecodedTextEvent, Finished, SampledDelta,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::output::ChatOutputProcessor;
|
||||
@@ -59,6 +61,21 @@ fn finished() -> Finished {
|
||||
}
|
||||
}
|
||||
|
||||
fn decoded_tokens(
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText::default(),
|
||||
sampled: SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_events(
|
||||
processor: HarmonyChatOutputProcessor,
|
||||
events: Vec<DecodedTextEvent>,
|
||||
@@ -96,12 +113,7 @@ fn interrupted_final_message_is_preserved() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -135,12 +147,7 @@ fn eos_flush_preserves_trailing_replacement_text() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -157,12 +164,7 @@ fn interrupted_analysis_message_is_preserved() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..tokens.len() - 1].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..tokens.len() - 1].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -196,12 +198,7 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -225,12 +222,7 @@ fn multiple_messages_get_newline_separators() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens, None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -256,18 +248,8 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
|
||||
HarmonyChatOutputProcessor::new(&request_with_tools()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[..midpoint].to_vec(),
|
||||
logprobs: None,
|
||||
finished: None,
|
||||
},
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens[midpoint..].to_vec(),
|
||||
logprobs: None,
|
||||
finished: Some(finished()),
|
||||
},
|
||||
decoded_tokens(tokens[..midpoint].to_vec(), None, None),
|
||||
decoded_tokens(tokens[midpoint..].to_vec(), None, Some(finished())),
|
||||
],
|
||||
));
|
||||
|
||||
@@ -303,14 +285,13 @@ fn semantic_events_precede_same_update_logprobs() {
|
||||
HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap(),
|
||||
vec![
|
||||
decoded_start(),
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: tokens,
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
decoded_tokens(
|
||||
tokens,
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs { entries: vec![] }],
|
||||
}),
|
||||
finished: Some(finished()),
|
||||
},
|
||||
Some(finished()),
|
||||
),
|
||||
],
|
||||
));
|
||||
|
||||
|
||||
@@ -784,15 +784,14 @@ fn decoded_completion_stream(
|
||||
if chunks.is_empty() {
|
||||
events.push({
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
decoded: vllm_text::DecodedText::default(),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: Default::default(),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}
|
||||
});
|
||||
} else {
|
||||
@@ -805,10 +804,12 @@ fn decoded_completion_stream(
|
||||
ec_transfer_params: None,
|
||||
});
|
||||
events.push(DecodedTextEvent::TextDelta {
|
||||
delta: chunk.delta,
|
||||
token_ids: chunk.token_ids,
|
||||
logprobs: None,
|
||||
finished,
|
||||
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
|
||||
sampled: vllm_text::SampledDelta {
|
||||
token_ids: chunk.token_ids,
|
||||
logprobs: None,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -864,7 +865,7 @@ fn incremental_decode_chunks(
|
||||
for token_id in chunk_token_ids.iter().copied() {
|
||||
decoder.push_token(token_id)?;
|
||||
while let Some(chunk) = decoder.next_chunk() {
|
||||
delta.push_str(&chunk);
|
||||
delta.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
chunks.push(DecodedCompletionChunk {
|
||||
@@ -876,10 +877,10 @@ fn incremental_decode_chunks(
|
||||
let (last_chunk, _) = decoder.flush(None)?;
|
||||
if let Some(last_chunk) = last_chunk {
|
||||
if let Some(delta) = chunks.last_mut() {
|
||||
delta.delta.push_str(&last_chunk);
|
||||
delta.delta.push_str(&last_chunk.text);
|
||||
} else {
|
||||
chunks.push(DecodedCompletionChunk {
|
||||
delta: last_chunk,
|
||||
delta: last_chunk.text,
|
||||
token_ids: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use tracing::{Span, info, info_span, warn};
|
||||
use tracing_futures::Instrument as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_llm::current_unix_timestamp_secs;
|
||||
use vllm_text::{DecodedTextEvent, Prompt, TextOutputStreamExt as _, TextRequest};
|
||||
use vllm_text::{DecodedTextEvent, Prompt, SampledDelta, TextOutputStreamExt as _, TextRequest};
|
||||
|
||||
use super::convert::{self, ResponseOpts};
|
||||
use super::{InferenceServer, pb};
|
||||
@@ -264,17 +264,20 @@ impl pb::inference_server::Inference for InferenceServiceImpl {
|
||||
})
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
}) => Ok(pb::GenerateResponse {
|
||||
prompt_info: None,
|
||||
outputs: Some(convert::to_sequence_output(
|
||||
&delta,
|
||||
&decoded.text,
|
||||
&token_ids,
|
||||
logprobs.as_ref(),
|
||||
finished.as_ref(),
|
||||
finished.as_deref(),
|
||||
&response_opts,
|
||||
)),
|
||||
}),
|
||||
|
||||
@@ -23,7 +23,7 @@ use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, SampledDelta, TextOutputStream,
|
||||
TextOutputStreamExt as _, TextRequest,
|
||||
};
|
||||
|
||||
@@ -317,11 +317,15 @@ async fn completion_chunk_stream(
|
||||
}
|
||||
}
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids,
|
||||
logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished,
|
||||
}) => {
|
||||
let delta = decoded.text;
|
||||
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
|
||||
// The one generated token is only used to drive the engine to a finished event,
|
||||
// so hide its delta and forward only the terminal finish/usage metadata.
|
||||
@@ -561,8 +565,8 @@ mod tests {
|
||||
use itertools::Itertools as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
|
||||
DecodedTokenLogprob, FinishReason, Finished,
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedText,
|
||||
DecodedTextEvent, DecodedTokenLogprob, FinishReason, Finished, SampledDelta,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -570,6 +574,22 @@ mod tests {
|
||||
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
fn decoded_delta(
|
||||
text: impl Into<String>,
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText::unattributed(text),
|
||||
sampled: SampledDelta {
|
||||
token_ids,
|
||||
logprobs,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"cmpl-1".to_string(),
|
||||
@@ -619,10 +639,10 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2, 3, 4, 5].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "h".to_string(),
|
||||
token_ids: vec![b'h' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
Ok(decoded_delta(
|
||||
"h",
|
||||
vec![b'h' as u32],
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
@@ -640,12 +660,12 @@ mod tests {
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: String::new(),
|
||||
token_ids: vec![b'!' as u32],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
None,
|
||||
)),
|
||||
Ok(decoded_delta(
|
||||
"",
|
||||
vec![b'!' as u32],
|
||||
Some(DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
@@ -663,7 +683,7 @@ mod tests {
|
||||
],
|
||||
}],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
@@ -675,7 +695,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -773,11 +793,11 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -787,7 +807,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
@@ -825,11 +845,11 @@ mod tests {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
@@ -839,7 +859,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
@@ -880,11 +900,11 @@ mod tests {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -894,7 +914,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -952,11 +972,11 @@ mod tests {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
@@ -966,7 +986,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
@@ -1026,11 +1046,11 @@ mod tests {
|
||||
}],
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
Ok(decoded_delta(
|
||||
" leaked",
|
||||
vec![3],
|
||||
None,
|
||||
Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
@@ -1040,7 +1060,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
)),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
|
||||
@@ -19,7 +19,8 @@ pub use lower::{
|
||||
};
|
||||
pub use output::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
|
||||
DecodedText, DecodedTextEvent, DecodedTokenLogprob, Finished, SampledDelta, TextDecodeOptions,
|
||||
TextOutputStreamExt, TokenAnchor, TokenAttribution,
|
||||
};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
|
||||
use trait_set::trait_set;
|
||||
|
||||
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
|
||||
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
|
||||
|
||||
use super::logprobs::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, decode_logprobs, decode_prompt_logprobs,
|
||||
@@ -52,6 +52,19 @@ pub struct Finished {
|
||||
pub ec_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Sample metadata emitted by one engine output update.
|
||||
///
|
||||
/// Token IDs and logprobs share the engine-update clock and align by generated
|
||||
/// token position. They may arrive before the corresponding decoded text is
|
||||
/// visible or alongside decoded text attributed to earlier updates.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct SampledDelta {
|
||||
/// Token IDs produced by this engine update, after stop-string truncation.
|
||||
pub token_ids: Vec<u32>,
|
||||
/// Per-position logprobs aligned with `token_ids`, when requested.
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
}
|
||||
|
||||
/// Internal decoded-text event emitted before higher-level assistant
|
||||
/// adaptation.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -68,25 +81,25 @@ pub enum DecodedTextEvent {
|
||||
/// remaining prompt positions.
|
||||
prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
},
|
||||
/// A delta of text has been decoded, optionally alongside token-position
|
||||
/// logprobs.
|
||||
/// Decoded text and sampled metadata observed during one output update.
|
||||
///
|
||||
/// `delta` is the newly visible decoded text fragment for this update.
|
||||
/// `decoded` follows the decoder clock. It may be empty while tokens remain
|
||||
/// undecodable, and may later contain token attributions from earlier
|
||||
/// engine updates. Tokens suppressed before decoding are absent.
|
||||
///
|
||||
/// `logprobs` covers the newly generated token positions from the same
|
||||
/// update, but is not guaranteed to align with `delta` by character
|
||||
/// span. One update may carry token logprobs but no newly visible text
|
||||
/// yet, and one visible text fragment may reflect multiple token
|
||||
/// positions becoming decodable together.
|
||||
/// `sampled` follows the engine-update clock. Its token IDs and logprobs
|
||||
/// align with each other. `decoded.attributions` has an independent decoder
|
||||
/// clock and positional sequence.
|
||||
///
|
||||
/// Upper-level may further parse `delta` as reasoning or tool calls.
|
||||
/// Upper-level may further parse `decoded.text` as reasoning or tool calls.
|
||||
///
|
||||
/// When `finished` is `Some`, this is the terminal event for the request.
|
||||
TextDelta {
|
||||
delta: String,
|
||||
token_ids: Vec<u32>,
|
||||
logprobs: Option<DecodedLogprobs>,
|
||||
finished: Option<Finished>,
|
||||
decoded: DecodedText,
|
||||
sampled: SampledDelta,
|
||||
/// Heap-backed because terminal metadata is large and appears once per
|
||||
/// request, while every streaming event carries this enum variant.
|
||||
finished: Option<Box<Finished>>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -104,9 +117,9 @@ pub async fn decoded_text_event_stream(
|
||||
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
|
||||
let mut prompt_token_count = 0_usize;
|
||||
let mut cached_token_count = 0_usize;
|
||||
let mut token_ids = Vec::new();
|
||||
let mut sampled_token_ids = Vec::new();
|
||||
let mut output_token_count: usize = 0;
|
||||
let mut logprobs: Option<DecodedLogprobs> = None;
|
||||
let mut sampled_logprobs: Option<DecodedLogprobs> = None;
|
||||
|
||||
while let Some(next) = raw_stream.next().await {
|
||||
let output = next?;
|
||||
@@ -168,7 +181,7 @@ pub async fn decoded_text_event_stream(
|
||||
&output.token_ids
|
||||
};
|
||||
|
||||
let mut delta: Option<String> = None;
|
||||
let mut decoded = DecodedText::default();
|
||||
let mut truncate_output_to = None;
|
||||
let mut truncate_tokens_to = None;
|
||||
for (tok_idx, &token_id) in decodable_token_ids.iter().enumerate() {
|
||||
@@ -189,12 +202,9 @@ pub async fn decoded_text_event_stream(
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: avoid generating a chunk every time a token is pushed
|
||||
if intermediate && let Some(chunk) = decoder.next_chunk() {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
decoded.append(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,9 +233,9 @@ pub async fn decoded_text_event_stream(
|
||||
.transpose()?;
|
||||
|
||||
if !intermediate {
|
||||
token_ids.extend(&new_token_ids);
|
||||
sampled_token_ids.extend(&new_token_ids);
|
||||
if let Some(dlp) = decoded_logprobs.as_ref() {
|
||||
logprobs
|
||||
sampled_logprobs
|
||||
.get_or_insert_with(|| DecodedLogprobs { positions: vec![] })
|
||||
.positions
|
||||
.extend_from_slice(&dlp.positions);
|
||||
@@ -234,22 +244,19 @@ pub async fn decoded_text_event_stream(
|
||||
|
||||
if let Some(reason) = finish_reason {
|
||||
// Flush any remaining buffered text.
|
||||
let (last_chunk, mut text) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = text.len();
|
||||
let full_text = tracing::enabled!(Level::TRACE).then(|| text.clone());
|
||||
|
||||
if intermediate {
|
||||
let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?;
|
||||
let text_len = full_decoded.text.len();
|
||||
let full_text = tracing::enabled!(Level::TRACE).then(|| full_decoded.text.clone());
|
||||
let decoded = if intermediate {
|
||||
if let Some(chunk) = last_chunk {
|
||||
if let Some(delta_str) = delta.as_mut() {
|
||||
delta_str.push_str(&chunk);
|
||||
} else {
|
||||
delta = Some(chunk);
|
||||
}
|
||||
decoded.append(chunk);
|
||||
}
|
||||
token_ids = new_token_ids;
|
||||
logprobs = decoded_logprobs;
|
||||
text = delta.unwrap_or_default();
|
||||
}
|
||||
sampled_token_ids = new_token_ids;
|
||||
sampled_logprobs = decoded_logprobs;
|
||||
decoded
|
||||
} else {
|
||||
full_decoded
|
||||
};
|
||||
|
||||
debug!(
|
||||
finish_reason = ?reason,
|
||||
@@ -269,10 +276,12 @@ pub async fn decoded_text_event_stream(
|
||||
}
|
||||
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: text,
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished: Some(Finished {
|
||||
decoded,
|
||||
sampled: SampledDelta {
|
||||
token_ids: sampled_token_ids,
|
||||
logprobs: sampled_logprobs,
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
@@ -281,7 +290,7 @@ pub async fn decoded_text_event_stream(
|
||||
finish_reason: reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
}),
|
||||
})),
|
||||
})
|
||||
.await;
|
||||
return Ok(());
|
||||
@@ -289,9 +298,11 @@ pub async fn decoded_text_event_stream(
|
||||
|
||||
if intermediate {
|
||||
y.yield_ok(DecodedTextEvent::TextDelta {
|
||||
delta: delta.unwrap_or_default(),
|
||||
token_ids: new_token_ids,
|
||||
logprobs: decoded_logprobs,
|
||||
decoded,
|
||||
sampled: SampledDelta {
|
||||
token_ids: new_token_ids,
|
||||
logprobs: decoded_logprobs,
|
||||
},
|
||||
finished: None,
|
||||
})
|
||||
.await;
|
||||
@@ -338,10 +349,10 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::{Stream, stream};
|
||||
use futures::{Stream, StreamExt as _, stream};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_llm::GenerateOutput;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer};
|
||||
|
||||
use super::*;
|
||||
use crate::output::TextOutputStreamExt as _;
|
||||
@@ -378,6 +389,129 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn visible(token_id: u32, byte_offset: u32) -> TokenAttribution {
|
||||
TokenAttribution {
|
||||
token_id,
|
||||
anchor: TokenAnchor::Visible { byte_offset },
|
||||
}
|
||||
}
|
||||
|
||||
fn text_delta(
|
||||
text: impl Into<String>,
|
||||
attributions: impl IntoIterator<Item = TokenAttribution>,
|
||||
sampled_token_ids: Vec<u32>,
|
||||
finished: Option<Finished>,
|
||||
) -> DecodedTextEvent {
|
||||
DecodedTextEvent::TextDelta {
|
||||
decoded: DecodedText {
|
||||
text: text.into(),
|
||||
attributions: attributions.into_iter().collect(),
|
||||
},
|
||||
sampled: SampledDelta {
|
||||
token_ids: sampled_token_ids,
|
||||
logprobs: None,
|
||||
},
|
||||
finished: finished.map(Box::new),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_intermediate(
|
||||
outputs: Vec<GenerateOutput>,
|
||||
decode_options: TextDecodeOptions,
|
||||
) -> Vec<DecodedTextEvent> {
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
decoded_text_event_stream(
|
||||
"test".into(),
|
||||
tokenizer,
|
||||
stream::iter(outputs.into_iter().map(Ok)),
|
||||
decode_options,
|
||||
true,
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decoded_and_sampled_clocks_are_independent() {
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
outputs: Vec<GenerateOutput>,
|
||||
expected: Vec<DecodedTextEvent>,
|
||||
}
|
||||
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let cases = [
|
||||
Case {
|
||||
name: "utf8 byte fallback resolves across engine updates",
|
||||
outputs: vec![
|
||||
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
|
||||
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
|
||||
],
|
||||
expected: vec![
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids: Arc::clone(&prompt),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
text_delta("", [], vec![0xe4], None),
|
||||
text_delta(
|
||||
"你",
|
||||
[visible(0xe4, 0), visible(0xbd, 0), visible(0xa0, 0)],
|
||||
vec![0xbd, 0xa0],
|
||||
Some(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 3,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
},
|
||||
Case {
|
||||
name: "terminal stop token remains sampled and bypasses decoder",
|
||||
outputs: vec![GenerateOutput::for_test(
|
||||
Some(Arc::clone(&prompt)),
|
||||
ascii_tokens("a!"),
|
||||
Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))),
|
||||
)],
|
||||
expected: vec![
|
||||
DecodedTextEvent::Start {
|
||||
prompt_token_ids: Arc::clone(&prompt),
|
||||
prompt_logprobs: None,
|
||||
},
|
||||
text_delta(
|
||||
"a",
|
||||
[visible(b'a' as u32, 0)],
|
||||
ascii_tokens("a!"),
|
||||
Some(Finished {
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(
|
||||
b'!' as u32,
|
||||
))),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
for case in cases {
|
||||
let actual = run_intermediate(case.outputs, TextDecodeOptions::default()).await;
|
||||
assert_eq!(actual, case.expected, "{}", case.name);
|
||||
}
|
||||
}
|
||||
|
||||
struct DropRecordingStream {
|
||||
next: Option<vllm_llm::Result<GenerateOutput>>,
|
||||
dropped_cause: Arc<Mutex<Option<AbortCause>>>,
|
||||
|
||||
@@ -3,10 +3,13 @@
|
||||
|
||||
//! Output processing helpers shared by text and chat layers.
|
||||
|
||||
pub use decoded::{DecodedTextEvent, Finished, TextDecodeOptions, decoded_text_event_stream};
|
||||
pub use decoded::{
|
||||
DecodedTextEvent, Finished, SampledDelta, TextDecodeOptions, decoded_text_event_stream,
|
||||
};
|
||||
pub use logprobs::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob,
|
||||
};
|
||||
pub use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution};
|
||||
|
||||
mod decoded;
|
||||
mod logprobs;
|
||||
@@ -58,13 +61,16 @@ impl<T: TextOutputStream> T {
|
||||
prompt_token_ids = start_prompt_token_ids;
|
||||
}
|
||||
DecodedTextEvent::TextDelta {
|
||||
delta,
|
||||
token_ids: delta_token_ids,
|
||||
logprobs: mut delta_logprobs,
|
||||
decoded,
|
||||
sampled:
|
||||
SampledDelta {
|
||||
token_ids: delta_token_ids,
|
||||
logprobs: mut delta_logprobs,
|
||||
},
|
||||
finished,
|
||||
} => {
|
||||
if let Some(c) = collected.as_mut() {
|
||||
c.text.push_str(&delta);
|
||||
c.text.push_str(&decoded.text);
|
||||
c.token_ids.extend(delta_token_ids);
|
||||
if let Some(dlp) = delta_logprobs.as_mut() {
|
||||
if let Some(lp) = c.logprobs.as_mut() {
|
||||
@@ -75,7 +81,7 @@ impl<T: TextOutputStream> T {
|
||||
}
|
||||
} else {
|
||||
collected = Some(CollectedTextOutput {
|
||||
text: delta,
|
||||
text: decoded.text,
|
||||
prompt_token_ids: Arc::clone(&prompt_token_ids),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
logprobs: delta_logprobs,
|
||||
@@ -134,29 +140,31 @@ mod tests {
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "bc".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
decoded: DecodedText::unattributed("bc"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "a".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "bc".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
@@ -165,7 +173,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -219,62 +227,66 @@ mod tests {
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "he".to_string(),
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
decoded: DecodedText::unattributed("he"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![1, 2],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "h".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "e".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: "llo".to_string(),
|
||||
token_ids: vec![3, 4, 5],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
decoded: DecodedText::unattributed("llo"),
|
||||
sampled: SampledDelta {
|
||||
token_ids: vec![3, 4, 5],
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
positions: vec![
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.3,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "l".to_string(),
|
||||
logprob: -0.4,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 0,
|
||||
token: "o".to_string(),
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
}],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
finished: Some(Box::new(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 5,
|
||||
@@ -283,7 +295,7 @@ mod tests {
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ riptoken.workspace = true
|
||||
rustc-hash.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
smallvec.workspace = true
|
||||
tekken.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
use crate::{Result, Tokenizer};
|
||||
|
||||
mod attribution;
|
||||
|
||||
use attribution::AttributedTextBuffer;
|
||||
pub use attribution::{DecodedText, TokenAnchor, TokenAttribution};
|
||||
|
||||
/// Stateful incremental decoder that emits text chunks one token at a time.
|
||||
pub trait IncrementalDecoder: Send {
|
||||
/// Push one generated token and return how many new string bytes were
|
||||
@@ -12,13 +15,16 @@ pub trait IncrementalDecoder: Send {
|
||||
fn push_token(&mut self, token_id: u32) -> Result<usize>;
|
||||
|
||||
/// Consume any text which is currently ready.
|
||||
fn next_chunk(&mut self) -> Option<String>;
|
||||
fn next_chunk(&mut self) -> Option<DecodedText>;
|
||||
|
||||
/// Flush any remaining buffered text that has not yet been emitted.
|
||||
///
|
||||
/// Called after the final generated token to force out buffered/incomplete
|
||||
/// fragments.
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
|
||||
fn flush(
|
||||
&mut self,
|
||||
truncate_output_to: Option<usize>,
|
||||
) -> Result<(Option<DecodedText>, DecodedText)>;
|
||||
|
||||
/// Return cumulative decoded text so far.
|
||||
fn output(&self) -> &str;
|
||||
@@ -36,8 +42,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
|
||||
prefix: String,
|
||||
prefix_index: usize,
|
||||
prefix_seeded: bool,
|
||||
cumulative_output: String,
|
||||
output_index: usize,
|
||||
decoded: AttributedTextBuffer,
|
||||
}
|
||||
|
||||
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
|
||||
@@ -55,8 +60,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
|
||||
prefix: String::new(),
|
||||
prefix_index: 0,
|
||||
prefix_seeded: prompt_token_ids.is_empty(),
|
||||
cumulative_output: String::new(),
|
||||
output_index: 0,
|
||||
decoded: AttributedTextBuffer::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,6 +72,14 @@ const SAFE_SUFFIX_MIN: usize = 4;
|
||||
const SAFE_SUFFIX_MAX: usize = 6;
|
||||
|
||||
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
|
||||
fn record_token(&mut self, token_id: u32) {
|
||||
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
|
||||
self.decoded.record_zero_width_token(token_id);
|
||||
} else {
|
||||
self.decoded.record_pending_token(token_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode prompt-only context for prefix seeding.
|
||||
///
|
||||
/// Prompt ids may come from the model vocabulary rather than the local
|
||||
@@ -132,6 +144,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
}
|
||||
|
||||
self.ids.push(token_id);
|
||||
self.record_token(token_id);
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
|
||||
@@ -139,49 +152,48 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
}
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
|
||||
self.cumulative_output.push_str(new_chunk);
|
||||
self.decoded.append_visible_text(new_chunk);
|
||||
self.ids.drain(..self.prefix_index);
|
||||
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
self.prefix_index = self.ids.len();
|
||||
Ok(new_chunk.len())
|
||||
}
|
||||
|
||||
fn next_chunk(&mut self) -> Option<String> {
|
||||
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
|
||||
fn next_chunk(&mut self) -> Option<DecodedText> {
|
||||
let cutoff = self.decoded.len().saturating_sub(self.min_bytes_to_buffer);
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
let cutoff = self.cumulative_output.floor_char_boundary(cutoff);
|
||||
(cutoff > self.output_index).then(|| {
|
||||
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
|
||||
self.output_index = cutoff;
|
||||
chunk
|
||||
})
|
||||
let cutoff = self.decoded.text().floor_char_boundary(cutoff);
|
||||
self.decoded.take_ready(cutoff)
|
||||
}
|
||||
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
|
||||
fn flush(
|
||||
&mut self,
|
||||
truncate_output_to: Option<usize>,
|
||||
) -> Result<(Option<DecodedText>, DecodedText)> {
|
||||
// If the prefix was never seeded (no push_token was called), `ids`
|
||||
// holds only prompt context — decoding it would re-emit prompt text.
|
||||
if self.prefix_seeded && !self.ids.is_empty() {
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
self.cumulative_output
|
||||
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
|
||||
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
|
||||
if !new_chunk.is_empty() {
|
||||
self.decoded.append_visible_text(new_chunk);
|
||||
}
|
||||
}
|
||||
self.decoded.resolve_pending_zero_width();
|
||||
self.ids.clear();
|
||||
self.prefix.clear();
|
||||
self.prefix_index = 0;
|
||||
self.prefix_seeded = true;
|
||||
if let Some(truncate_output_to) = truncate_output_to {
|
||||
self.cumulative_output.truncate(truncate_output_to);
|
||||
self.decoded.truncate(truncate_output_to);
|
||||
}
|
||||
let last_chunk = (self.output_index < self.cumulative_output.len())
|
||||
.then(|| self.cumulative_output[self.output_index..].to_string());
|
||||
self.output_index = 0;
|
||||
Ok((last_chunk, take(&mut self.cumulative_output)))
|
||||
Ok(self.decoded.finish())
|
||||
}
|
||||
|
||||
fn output(&self) -> &str {
|
||||
&self.cumulative_output
|
||||
self.decoded.text()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,13 +257,13 @@ mod tests {
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
|
||||
assert_eq!(decoder.next_chunk().unwrap().text, "o");
|
||||
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
|
||||
assert_eq!(decoder.next_chunk().unwrap().text, "k");
|
||||
// All text already consumed via next_chunk
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "ok");
|
||||
assert_eq!(full_text.text, "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -288,6 +300,7 @@ mod tests {
|
||||
0 if !skip_special_tokens => text.push_str("<special>"),
|
||||
0 => {}
|
||||
1 => text.push('a'),
|
||||
2 => text.push('b'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -301,6 +314,10 @@ mod tests {
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -393,17 +410,17 @@ mod tests {
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
let input = b"Hello, world!";
|
||||
let mut full = String::new();
|
||||
let mut out = String::new();
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
full.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None); // all consumed via next_chunk
|
||||
assert_eq!(full, "Hello, world!");
|
||||
assert_eq!(full_text, "Hello, world!");
|
||||
assert_eq!(out, "Hello, world!");
|
||||
assert_eq!(full.text, "Hello, world!");
|
||||
}
|
||||
|
||||
/// Backend simulating non-monotonic decode where adding a token changes how
|
||||
@@ -470,15 +487,15 @@ mod tests {
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
chunks.push_str(&chunk);
|
||||
chunks.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
|
||||
assert_eq!(chunks, "Hel");
|
||||
// Flush returns the rest
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk.as_deref(), Some("lo!"));
|
||||
assert_eq!(full_text, "Hello!");
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk.unwrap().text, "lo!");
|
||||
assert_eq!(full.text, "Hello!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -493,14 +510,14 @@ mod tests {
|
||||
for byte in "你好A".bytes() {
|
||||
decoder.push_token(u32::from(byte)).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
out.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
out.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
assert_eq!(full_text, "你好A");
|
||||
assert_eq!(full.text, "你好A");
|
||||
assert_eq!(out, "你好A");
|
||||
}
|
||||
|
||||
@@ -515,9 +532,9 @@ mod tests {
|
||||
.collect();
|
||||
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
|
||||
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "");
|
||||
assert_eq!(full.text, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -526,8 +543,8 @@ mod tests {
|
||||
let prompt = vec![0xe4, 0xbd];
|
||||
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
|
||||
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "");
|
||||
assert_eq!(full.text, "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
/// Position of one generated token in decoded text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TokenAnchor {
|
||||
/// The token contributed to visible decoded text.
|
||||
Visible { byte_offset: u32 },
|
||||
/// The token produced no visible decoded bytes.
|
||||
ZeroWidth { byte_offset: u32 },
|
||||
}
|
||||
|
||||
impl TokenAnchor {
|
||||
fn byte_offset(self) -> u32 {
|
||||
match self {
|
||||
Self::Visible { byte_offset } | Self::ZeroWidth { byte_offset } => byte_offset,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a new anchor offset by `delta` bytes.
|
||||
#[must_use]
|
||||
pub(super) fn offset_by(self, delta: i64) -> Self {
|
||||
let byte_offset = i64::from(self.byte_offset())
|
||||
.checked_add(delta)
|
||||
.and_then(|offset| u32::try_from(offset).ok())
|
||||
.expect("token anchor byte offset out of range");
|
||||
match self {
|
||||
Self::Visible { .. } => Self::Visible { byte_offset },
|
||||
Self::ZeroWidth { .. } => Self::ZeroWidth { byte_offset },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One generated token and its position in decoded text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TokenAttribution {
|
||||
/// Token ID passed to [`super::IncrementalDecoder::push_token`].
|
||||
pub token_id: u32,
|
||||
/// Position of the token in the decoded text.
|
||||
pub anchor: TokenAnchor,
|
||||
}
|
||||
|
||||
/// Decoded text and the generated tokens attributed to it.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct DecodedText {
|
||||
/// Decoded UTF-8 text.
|
||||
pub text: String,
|
||||
/// One entry per generated token, in generation order.
|
||||
///
|
||||
/// Offsets are local to `text`. Visible offsets may repeat when multiple
|
||||
/// tokens jointly decode into one character. A zero-width offset may equal
|
||||
/// `text.len()`.
|
||||
/// Four inline records cover one complete UTF-8 byte-fallback sequence.
|
||||
pub attributions: SmallVec<[TokenAttribution; 4]>,
|
||||
}
|
||||
|
||||
impl DecodedText {
|
||||
/// Create decoded text without token attribution records. Mostly for testing purposes.
|
||||
pub fn unattributed(text: impl Into<String>) -> Self {
|
||||
Self {
|
||||
text: text.into(),
|
||||
attributions: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append another decoded fragment, rebasing its token anchors.
|
||||
pub fn append(&mut self, other: Self) {
|
||||
if self.text.is_empty() && self.attributions.is_empty() {
|
||||
*self = other;
|
||||
return;
|
||||
}
|
||||
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
let other_len = offset_as_u32(other.text.len());
|
||||
let _combined_len = byte_offset.checked_add(other_len).expect("decoded text exceeds 4 GiB");
|
||||
|
||||
self.text.push_str(&other.text);
|
||||
self.attributions.extend(other.attributions.into_iter().map(|attribution| {
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: attribution.anchor.offset_by(i64::from(byte_offset)),
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum PendingAnchor {
|
||||
Unresolved,
|
||||
Resolved(TokenAnchor),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct PendingAttribution {
|
||||
token_id: u32,
|
||||
anchor: PendingAnchor,
|
||||
}
|
||||
|
||||
/// Cumulative decoded text plus the attribution state needed to emit local
|
||||
/// chunks.
|
||||
#[derive(Default)]
|
||||
pub(super) struct AttributedTextBuffer {
|
||||
text: String,
|
||||
emitted_byte_offset: usize,
|
||||
attributions: Vec<PendingAttribution>,
|
||||
emitted_attribution_count: usize,
|
||||
pending_start: Option<usize>,
|
||||
}
|
||||
|
||||
impl AttributedTextBuffer {
|
||||
/// Return all decoded text accumulated so far.
|
||||
pub(super) fn text(&self) -> &str {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Return the decoded text length in bytes.
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.text.len()
|
||||
}
|
||||
|
||||
/// Record a token awaiting visible decoded output.
|
||||
pub(super) fn record_pending_token(&mut self, token_id: u32) {
|
||||
self.pending_start.get_or_insert(self.attributions.len());
|
||||
self.attributions.push(PendingAttribution {
|
||||
token_id,
|
||||
anchor: PendingAnchor::Unresolved,
|
||||
});
|
||||
}
|
||||
|
||||
/// Record a token that produced no visible decoded output.
|
||||
pub(super) fn record_zero_width_token(&mut self, token_id: u32) {
|
||||
self.attributions.push(PendingAttribution {
|
||||
token_id,
|
||||
anchor: PendingAnchor::Resolved(TokenAnchor::ZeroWidth {
|
||||
byte_offset: offset_as_u32(self.text.len()),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/// Append visible text and anchor pending tokens at its start.
|
||||
pub(super) fn append_visible_text(&mut self, text: &str) {
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
if let Some(pending_start) = self.pending_start.take() {
|
||||
for attribution in &mut self.attributions[pending_start..] {
|
||||
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
|
||||
attribution.anchor =
|
||||
PendingAnchor::Resolved(TokenAnchor::Visible { byte_offset });
|
||||
}
|
||||
}
|
||||
}
|
||||
self.text.push_str(text);
|
||||
}
|
||||
|
||||
/// Resolve remaining pending tokens at the current text boundary.
|
||||
pub(super) fn resolve_pending_zero_width(&mut self) {
|
||||
let Some(pending_start) = self.pending_start.take() else {
|
||||
return;
|
||||
};
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
for attribution in &mut self.attributions[pending_start..] {
|
||||
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
|
||||
attribution.anchor =
|
||||
PendingAnchor::Resolved(TokenAnchor::ZeroWidth { byte_offset });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate text and collapse removed anchors to the new boundary.
|
||||
pub(super) fn truncate(&mut self, byte_offset: usize) {
|
||||
self.text.truncate(byte_offset);
|
||||
let byte_offset = offset_as_u32(byte_offset);
|
||||
for attribution in &mut self.attributions {
|
||||
let PendingAnchor::Resolved(anchor) = &mut attribution.anchor else {
|
||||
continue;
|
||||
};
|
||||
if anchor.byte_offset() >= byte_offset {
|
||||
*anchor = TokenAnchor::ZeroWidth { byte_offset };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit decoded text and attributions ready through `cutoff`.
|
||||
pub(super) fn take_ready(&mut self, cutoff: usize) -> Option<DecodedText> {
|
||||
let chunk_start = self.emitted_byte_offset;
|
||||
let cutoff_u32 = offset_as_u32(cutoff);
|
||||
let mut attribution_end = self.emitted_attribution_count;
|
||||
|
||||
for attribution in &self.attributions[self.emitted_attribution_count..] {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
break;
|
||||
};
|
||||
let ready = match anchor {
|
||||
TokenAnchor::Visible { byte_offset } => byte_offset < cutoff_u32,
|
||||
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= cutoff_u32,
|
||||
};
|
||||
if !ready {
|
||||
break;
|
||||
}
|
||||
attribution_end += 1;
|
||||
}
|
||||
|
||||
if cutoff == chunk_start && attribution_end == self.emitted_attribution_count {
|
||||
return None;
|
||||
}
|
||||
|
||||
let chunk_start_u32 = offset_as_u32(chunk_start);
|
||||
let attributions = self.attributions[self.emitted_attribution_count..attribution_end]
|
||||
.iter()
|
||||
.map(|attribution| {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
unreachable!("ready attributions must be resolved")
|
||||
};
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: anchor.offset_by(-i64::from(chunk_start_u32)),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let text = self.text[chunk_start..cutoff].to_string();
|
||||
|
||||
self.emitted_byte_offset = cutoff;
|
||||
self.emitted_attribution_count = attribution_end;
|
||||
Some(DecodedText { text, attributions })
|
||||
}
|
||||
|
||||
/// Emit the final chunk and take the complete decoded output.
|
||||
pub(super) fn finish(&mut self) -> (Option<DecodedText>, DecodedText) {
|
||||
let last_chunk = self.take_ready(self.text.len());
|
||||
let attributions = take(&mut self.attributions)
|
||||
.into_iter()
|
||||
.map(|attribution| {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
unreachable!("finish requires every attribution to be resolved")
|
||||
};
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let full = DecodedText {
|
||||
text: take(&mut self.text),
|
||||
attributions,
|
||||
};
|
||||
self.emitted_byte_offset = 0;
|
||||
self.emitted_attribution_count = 0;
|
||||
self.pending_start = None;
|
||||
(last_chunk, full)
|
||||
}
|
||||
}
|
||||
|
||||
fn offset_as_u32(offset: usize) -> u32 {
|
||||
u32::try_from(offset).expect("decoded text exceeds 4 GiB")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,425 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use super::*;
|
||||
use crate::{Result, Tokenizer, test_utils::TestTokenizer};
|
||||
|
||||
const SPECIAL_TOKEN_ID: u32 = 0x100;
|
||||
|
||||
/// Backend with tokens that decode to whole text pieces.
|
||||
#[derive(Debug)]
|
||||
struct PieceBackend;
|
||||
|
||||
impl Tokenizer for PieceBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| match token_id {
|
||||
1 => "abcd",
|
||||
2 => "ab",
|
||||
3 => "<stop>",
|
||||
4 => "",
|
||||
_ => unreachable!("unexpected token id: {token_id}"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw-byte backend with one filterable special token.
|
||||
#[derive(Debug)]
|
||||
struct Utf8SpecialBackend;
|
||||
|
||||
impl Tokenizer for Utf8SpecialBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let mut bytes = Vec::new();
|
||||
for &token_id in token_ids {
|
||||
if token_id == SPECIAL_TOKEN_ID {
|
||||
if !skip_special_tokens {
|
||||
bytes.extend_from_slice(b"<special>");
|
||||
}
|
||||
} else {
|
||||
bytes.push(token_id as u8);
|
||||
}
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == SPECIAL_TOKEN_ID
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer_with_special_token() -> TestTokenizer {
|
||||
TestTokenizer::new().with_special_token("<special>", SPECIAL_TOKEN_ID)
|
||||
}
|
||||
|
||||
fn visible(byte_offset: u32) -> TokenAnchor {
|
||||
TokenAnchor::Visible { byte_offset }
|
||||
}
|
||||
|
||||
fn zero_width(byte_offset: u32) -> TokenAnchor {
|
||||
TokenAnchor::ZeroWidth { byte_offset }
|
||||
}
|
||||
|
||||
fn decoded(text: &str, token_ids: &[u32], anchors: &[TokenAnchor]) -> DecodedText {
|
||||
assert_eq!(token_ids.len(), anchors.len());
|
||||
DecodedText {
|
||||
text: text.to_string(),
|
||||
attributions: token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(anchors.iter().copied())
|
||||
.map(|(token_id, anchor)| TokenAttribution { token_id, anchor })
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_anchor_offset_by_preserves_anchor_kind() {
|
||||
for (anchor, delta, expected) in [
|
||||
(visible(3), -2, visible(1)),
|
||||
(zero_width(1), 4, zero_width(5)),
|
||||
] {
|
||||
assert_eq!(anchor.offset_by(delta), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_rebases_attributions() {
|
||||
let mut combined = decoded("a", &[1], &[visible(0)]);
|
||||
combined.append(decoded(
|
||||
"你",
|
||||
&[2, 3, 4, 5],
|
||||
&[visible(0), visible(0), visible(0), zero_width(3)],
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
combined,
|
||||
decoded(
|
||||
"a你",
|
||||
&[1, 2, 3, 4, 5],
|
||||
&[
|
||||
visible(0),
|
||||
visible(1),
|
||||
visible(1),
|
||||
visible(1),
|
||||
zero_width(4),
|
||||
],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_moves_allocations_into_empty_receiver() {
|
||||
let source = decoded(
|
||||
"hello",
|
||||
&[1, 2, 3, 4, 5],
|
||||
&[visible(0), visible(1), visible(2), visible(3), visible(4)],
|
||||
);
|
||||
let text_ptr = source.text.as_ptr();
|
||||
let attributions_ptr = source.attributions.as_ptr();
|
||||
let mut combined = DecodedText::default();
|
||||
|
||||
combined.append(source);
|
||||
|
||||
assert_eq!(combined.text.as_ptr(), text_ptr);
|
||||
assert_eq!(combined.attributions.as_ptr(), attributions_ptr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_preserves_zero_width_only_receiver() {
|
||||
let mut combined = decoded("", &[1], &[zero_width(0)]);
|
||||
combined.append(decoded("a", &[2], &[visible(0)]));
|
||||
|
||||
assert_eq!(
|
||||
combined,
|
||||
decoded("a", &[1, 2], &[zero_width(0), visible(0)])
|
||||
);
|
||||
}
|
||||
|
||||
struct AttributionCase<'a> {
|
||||
tokenizer: &'a dyn Tokenizer,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
token_ids: Vec<u32>,
|
||||
skip_special_tokens: bool,
|
||||
min_bytes_to_buffer: usize,
|
||||
truncate_output_to: Option<usize>,
|
||||
expected_chunks: Vec<DecodedText>,
|
||||
expected_full: DecodedText,
|
||||
}
|
||||
|
||||
fn run_attribution_case(case: AttributionCase<'_>) {
|
||||
let mut decoder = case.tokenizer.create_decode_stream(
|
||||
&case.prompt_token_ids,
|
||||
case.skip_special_tokens,
|
||||
case.min_bytes_to_buffer,
|
||||
);
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
for token_id in &case.token_ids {
|
||||
decoder.push_token(*token_id).unwrap();
|
||||
while let Some(chunk) = decoder.next_chunk() {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
let (remaining, full) = decoder.flush(case.truncate_output_to).unwrap();
|
||||
if let Some(remaining) = remaining {
|
||||
chunks.push(remaining);
|
||||
}
|
||||
|
||||
assert_eq!(chunks, case.expected_chunks, "chunks");
|
||||
assert_eq!(full, case.expected_full, "full output");
|
||||
assert_eq!(
|
||||
full.attributions.len(),
|
||||
case.token_ids.len(),
|
||||
"one attribution per pushed token"
|
||||
);
|
||||
assert_eq!(
|
||||
full.attributions
|
||||
.iter()
|
||||
.map(|attribution| attribution.token_id)
|
||||
.collect::<Vec<_>>(),
|
||||
case.token_ids,
|
||||
"attribution preserves pushed token IDs"
|
||||
);
|
||||
|
||||
let mut reconstructed_text = String::new();
|
||||
let mut reconstructed_attributions = Vec::new();
|
||||
for chunk in &chunks {
|
||||
let chunk_start = reconstructed_text.len() as u32;
|
||||
reconstructed_text.push_str(&chunk.text);
|
||||
reconstructed_attributions.extend(chunk.attributions.iter().map(|attribution| {
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: attribution.anchor.offset_by(i64::from(chunk_start)),
|
||||
}
|
||||
}));
|
||||
}
|
||||
assert_eq!(
|
||||
reconstructed_text, full.text,
|
||||
"chunk text reconstructs full text"
|
||||
);
|
||||
assert_eq!(
|
||||
reconstructed_attributions.as_slice(),
|
||||
full.attributions.as_slice(),
|
||||
"chunk attributions reconstruct full attributions"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_ascii_tokens_have_one_visible_anchor_each() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![b'o' as u32, b'k' as u32],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![
|
||||
decoded("o", &[b'o' as u32], &[visible(0)]),
|
||||
decoded("k", &[b'k' as u32], &[visible(0)]),
|
||||
],
|
||||
expected_full: decoded("ok", &[b'o' as u32, b'k' as u32], &[visible(0), visible(1)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_byte_fallback_tokens_share_one_visible_anchor() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, 0xbd, 0xa0],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded(
|
||||
"你",
|
||||
&[0xe4, 0xbd, 0xa0],
|
||||
&[visible(0), visible(0), visible(0)],
|
||||
)],
|
||||
expected_full: decoded(
|
||||
"你",
|
||||
&[0xe4, 0xbd, 0xa0],
|
||||
&[visible(0), visible(0), visible(0)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_holdback_splits_text_without_repeating_token() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![1],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 2,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("ab", &[1], &[visible(0)]), decoded("cd", &[], &[])],
|
||||
expected_full: decoded("abcd", &[1], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_filtered_special_token_is_zero_width_at_its_byte_boundary() {
|
||||
let tokenizer = tokenizer_with_special_token();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
|
||||
skip_special_tokens: true,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![
|
||||
decoded("a", &[b'a' as u32], &[visible(0)]),
|
||||
decoded("", &[SPECIAL_TOKEN_ID], &[zero_width(0)]),
|
||||
decoded("b", &[b'b' as u32], &[visible(0)]),
|
||||
],
|
||||
expected_full: decoded(
|
||||
"ab",
|
||||
&[b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
|
||||
&[visible(0), zero_width(1), visible(1)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_zero_width_preserves_order_inside_byte_fallback_group() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &Utf8SpecialBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
skip_special_tokens: true,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded(
|
||||
"你",
|
||||
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
&[visible(0), zero_width(0), visible(0), visible(0)],
|
||||
)],
|
||||
expected_full: decoded(
|
||||
"你",
|
||||
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
&[visible(0), zero_width(0), visible(0), visible(0)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_retained_special_token_has_visible_anchor() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &Utf8SpecialBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![SPECIAL_TOKEN_ID],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)])],
|
||||
expected_full: decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_empty_decode_resolves_to_zero_width_on_flush() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![4],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("", &[4], &[zero_width(0)])],
|
||||
expected_full: decoded("", &[4], &[zero_width(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, 0xbd],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)])],
|
||||
expected_full: decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_truncation_converts_fully_removed_token_to_zero_width() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![2, 3],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 32,
|
||||
truncate_output_to: Some(2),
|
||||
expected_chunks: vec![decoded("ab", &[2, 3], &[visible(0), zero_width(2)])],
|
||||
expected_full: decoded("ab", &[2, 3], &[visible(0), zero_width(2)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_truncation_inside_token_retains_first_byte_anchor() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![1],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 32,
|
||||
truncate_output_to: Some(2),
|
||||
expected_chunks: vec![decoded("ab", &[1], &[visible(0)])],
|
||||
expected_full: decoded("ab", &[1], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_excludes_prompt_context() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![b'H' as u32, b'i' as u32],
|
||||
token_ids: vec![b'!' as u32],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("!", &[b'!' as u32], &[visible(0)])],
|
||||
expected_full: decoded("!", &[b'!' as u32], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
@@ -17,7 +17,7 @@ mod tiktoken;
|
||||
|
||||
pub use error::{Result, TokenizerError};
|
||||
pub use hf::HuggingFaceTokenizer;
|
||||
pub use incremental::IncrementalDecoder;
|
||||
pub use incremental::{DecodedText, IncrementalDecoder, TokenAnchor, TokenAttribution};
|
||||
pub use tekken::TekkenTokenizer;
|
||||
pub use tiktoken::TiktokenTokenizer;
|
||||
|
||||
|
||||
@@ -964,16 +964,16 @@ mod tests {
|
||||
for &id in &ids {
|
||||
decoder.push_token(id).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
|
||||
assert_eq!(output, text);
|
||||
assert_eq!(full_text, text);
|
||||
assert_eq!(full.text, text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -991,16 +991,16 @@ mod tests {
|
||||
for &id in &ids {
|
||||
decoder.push_token(id).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
|
||||
assert_eq!(output, text);
|
||||
assert_eq!(full_text, text);
|
||||
assert_eq!(full.text, text);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user