[Rust Frontend] Attribute decoded text to tokens (#52910)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-01 22:25:31 +00:00
committed by GitHub
parent 259a209bfa
commit b911fe85c4
17 changed files with 1182 additions and 320 deletions
+1
View File
@@ -14,6 +14,7 @@ riptoken.workspace = true
rustc-hash.workspace = true
serde.workspace = true
serde_json.workspace = true
smallvec.workspace = true
tekken.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
+63 -46
View File
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::mem::take;
use crate::{Result, Tokenizer};
mod attribution;
use attribution::AttributedTextBuffer;
pub use attribution::{DecodedText, TokenAnchor, TokenAttribution};
/// Stateful incremental decoder that emits text chunks one token at a time.
pub trait IncrementalDecoder: Send {
/// Push one generated token and return how many new string bytes were
@@ -12,13 +15,16 @@ pub trait IncrementalDecoder: Send {
fn push_token(&mut self, token_id: u32) -> Result<usize>;
/// Consume any text which is currently ready.
fn next_chunk(&mut self) -> Option<String>;
fn next_chunk(&mut self) -> Option<DecodedText>;
/// Flush any remaining buffered text that has not yet been emitted.
///
/// Called after the final generated token to force out buffered/incomplete
/// fragments.
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
fn flush(
&mut self,
truncate_output_to: Option<usize>,
) -> Result<(Option<DecodedText>, DecodedText)>;
/// Return cumulative decoded text so far.
fn output(&self) -> &str;
@@ -36,8 +42,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
prefix: String,
prefix_index: usize,
prefix_seeded: bool,
cumulative_output: String,
output_index: usize,
decoded: AttributedTextBuffer,
}
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
@@ -55,8 +60,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
prefix: String::new(),
prefix_index: 0,
prefix_seeded: prompt_token_ids.is_empty(),
cumulative_output: String::new(),
output_index: 0,
decoded: AttributedTextBuffer::default(),
}
}
}
@@ -68,6 +72,14 @@ const SAFE_SUFFIX_MIN: usize = 4;
const SAFE_SUFFIX_MAX: usize = 6;
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
fn record_token(&mut self, token_id: u32) {
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
self.decoded.record_zero_width_token(token_id);
} else {
self.decoded.record_pending_token(token_id);
}
}
/// Decode prompt-only context for prefix seeding.
///
/// Prompt ids may come from the model vocabulary rather than the local
@@ -132,6 +144,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
self.ids.push(token_id);
self.record_token(token_id);
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
@@ -139,49 +152,48 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
// Ensure we split at a utf-8 char boundary.
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
self.cumulative_output.push_str(new_chunk);
self.decoded.append_visible_text(new_chunk);
self.ids.drain(..self.prefix_index);
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
self.prefix_index = self.ids.len();
Ok(new_chunk.len())
}
fn next_chunk(&mut self) -> Option<String> {
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
fn next_chunk(&mut self) -> Option<DecodedText> {
let cutoff = self.decoded.len().saturating_sub(self.min_bytes_to_buffer);
// Ensure we split at a utf-8 char boundary.
let cutoff = self.cumulative_output.floor_char_boundary(cutoff);
(cutoff > self.output_index).then(|| {
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
self.output_index = cutoff;
chunk
})
let cutoff = self.decoded.text().floor_char_boundary(cutoff);
self.decoded.take_ready(cutoff)
}
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
fn flush(
&mut self,
truncate_output_to: Option<usize>,
) -> Result<(Option<DecodedText>, DecodedText)> {
// If the prefix was never seeded (no push_token was called), `ids`
// holds only prompt context — decoding it would re-emit prompt text.
if self.prefix_seeded && !self.ids.is_empty() {
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
// Ensure we split at a utf-8 char boundary.
self.cumulative_output
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
if !new_chunk.is_empty() {
self.decoded.append_visible_text(new_chunk);
}
}
self.decoded.resolve_pending_zero_width();
self.ids.clear();
self.prefix.clear();
self.prefix_index = 0;
self.prefix_seeded = true;
if let Some(truncate_output_to) = truncate_output_to {
self.cumulative_output.truncate(truncate_output_to);
self.decoded.truncate(truncate_output_to);
}
let last_chunk = (self.output_index < self.cumulative_output.len())
.then(|| self.cumulative_output[self.output_index..].to_string());
self.output_index = 0;
Ok((last_chunk, take(&mut self.cumulative_output)))
Ok(self.decoded.finish())
}
fn output(&self) -> &str {
&self.cumulative_output
self.decoded.text()
}
}
@@ -245,13 +257,13 @@ mod tests {
let mut decoder = backend.create_decode_stream(&[], false, 0);
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
assert_eq!(decoder.next_chunk().unwrap().text, "o");
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
assert_eq!(decoder.next_chunk().unwrap().text, "k");
// All text already consumed via next_chunk
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "ok");
assert_eq!(full_text.text, "ok");
}
#[test]
@@ -288,6 +300,7 @@ mod tests {
0 if !skip_special_tokens => text.push_str("<special>"),
0 => {}
1 => text.push('a'),
2 => text.push('b'),
_ => {}
}
}
@@ -301,6 +314,10 @@ mod tests {
fn id_to_token(&self, _id: u32) -> Option<String> {
unreachable!()
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == 0
}
}
#[test]
@@ -393,17 +410,17 @@ mod tests {
let mut decoder = backend.create_decode_stream(&[], false, 0);
let input = b"Hello, world!";
let mut full = String::new();
let mut out = String::new();
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
full.push_str(&chunk);
out.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None); // all consumed via next_chunk
assert_eq!(full, "Hello, world!");
assert_eq!(full_text, "Hello, world!");
assert_eq!(out, "Hello, world!");
assert_eq!(full.text, "Hello, world!");
}
/// Backend simulating non-monotonic decode where adding a token changes how
@@ -470,15 +487,15 @@ mod tests {
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
chunks.push_str(&chunk);
chunks.push_str(&chunk.text);
}
}
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
assert_eq!(chunks, "Hel");
// Flush returns the rest
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk.as_deref(), Some("lo!"));
assert_eq!(full_text, "Hello!");
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk.unwrap().text, "lo!");
assert_eq!(full.text, "Hello!");
}
#[test]
@@ -493,14 +510,14 @@ mod tests {
for byte in "你好A".bytes() {
decoder.push_token(u32::from(byte)).unwrap();
if let Some(chunk) = decoder.next_chunk() {
out.push_str(&chunk);
out.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
out.push_str(&chunk);
out.push_str(&chunk.text);
}
assert_eq!(full_text, "你好A");
assert_eq!(full.text, "你好A");
assert_eq!(out, "你好A");
}
@@ -515,9 +532,9 @@ mod tests {
.collect();
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
assert_eq!(full.text, "");
}
#[test]
@@ -526,8 +543,8 @@ mod tests {
let prompt = vec![0xe4, 0xbd];
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
assert_eq!(full.text, "");
}
}
@@ -0,0 +1,261 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::mem::take;
use smallvec::SmallVec;
/// Position of one generated token in decoded text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TokenAnchor {
/// The token contributed to visible decoded text.
Visible { byte_offset: u32 },
/// The token produced no visible decoded bytes.
ZeroWidth { byte_offset: u32 },
}
impl TokenAnchor {
fn byte_offset(self) -> u32 {
match self {
Self::Visible { byte_offset } | Self::ZeroWidth { byte_offset } => byte_offset,
}
}
/// Return a new anchor offset by `delta` bytes.
#[must_use]
pub(super) fn offset_by(self, delta: i64) -> Self {
let byte_offset = i64::from(self.byte_offset())
.checked_add(delta)
.and_then(|offset| u32::try_from(offset).ok())
.expect("token anchor byte offset out of range");
match self {
Self::Visible { .. } => Self::Visible { byte_offset },
Self::ZeroWidth { .. } => Self::ZeroWidth { byte_offset },
}
}
}
/// One generated token and its position in decoded text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TokenAttribution {
/// Token ID passed to [`super::IncrementalDecoder::push_token`].
pub token_id: u32,
/// Position of the token in the decoded text.
pub anchor: TokenAnchor,
}
/// Decoded text and the generated tokens attributed to it.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct DecodedText {
/// Decoded UTF-8 text.
pub text: String,
/// One entry per generated token, in generation order.
///
/// Offsets are local to `text`. Visible offsets may repeat when multiple
/// tokens jointly decode into one character. A zero-width offset may equal
/// `text.len()`.
/// Four inline records cover one complete UTF-8 byte-fallback sequence.
pub attributions: SmallVec<[TokenAttribution; 4]>,
}
impl DecodedText {
/// Create decoded text without token attribution records. Mostly for testing purposes.
pub fn unattributed(text: impl Into<String>) -> Self {
Self {
text: text.into(),
attributions: SmallVec::new(),
}
}
/// Append another decoded fragment, rebasing its token anchors.
pub fn append(&mut self, other: Self) {
if self.text.is_empty() && self.attributions.is_empty() {
*self = other;
return;
}
let byte_offset = offset_as_u32(self.text.len());
let other_len = offset_as_u32(other.text.len());
let _combined_len = byte_offset.checked_add(other_len).expect("decoded text exceeds 4 GiB");
self.text.push_str(&other.text);
self.attributions.extend(other.attributions.into_iter().map(|attribution| {
TokenAttribution {
token_id: attribution.token_id,
anchor: attribution.anchor.offset_by(i64::from(byte_offset)),
}
}));
}
}
#[derive(Clone, Copy, Debug)]
enum PendingAnchor {
Unresolved,
Resolved(TokenAnchor),
}
#[derive(Clone, Copy, Debug)]
struct PendingAttribution {
token_id: u32,
anchor: PendingAnchor,
}
/// Cumulative decoded text plus the attribution state needed to emit local
/// chunks.
#[derive(Default)]
pub(super) struct AttributedTextBuffer {
text: String,
emitted_byte_offset: usize,
attributions: Vec<PendingAttribution>,
emitted_attribution_count: usize,
pending_start: Option<usize>,
}
impl AttributedTextBuffer {
/// Return all decoded text accumulated so far.
pub(super) fn text(&self) -> &str {
&self.text
}
/// Return the decoded text length in bytes.
pub(super) fn len(&self) -> usize {
self.text.len()
}
/// Record a token awaiting visible decoded output.
pub(super) fn record_pending_token(&mut self, token_id: u32) {
self.pending_start.get_or_insert(self.attributions.len());
self.attributions.push(PendingAttribution {
token_id,
anchor: PendingAnchor::Unresolved,
});
}
/// Record a token that produced no visible decoded output.
pub(super) fn record_zero_width_token(&mut self, token_id: u32) {
self.attributions.push(PendingAttribution {
token_id,
anchor: PendingAnchor::Resolved(TokenAnchor::ZeroWidth {
byte_offset: offset_as_u32(self.text.len()),
}),
});
}
/// Append visible text and anchor pending tokens at its start.
pub(super) fn append_visible_text(&mut self, text: &str) {
let byte_offset = offset_as_u32(self.text.len());
if let Some(pending_start) = self.pending_start.take() {
for attribution in &mut self.attributions[pending_start..] {
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
attribution.anchor =
PendingAnchor::Resolved(TokenAnchor::Visible { byte_offset });
}
}
}
self.text.push_str(text);
}
/// Resolve remaining pending tokens at the current text boundary.
pub(super) fn resolve_pending_zero_width(&mut self) {
let Some(pending_start) = self.pending_start.take() else {
return;
};
let byte_offset = offset_as_u32(self.text.len());
for attribution in &mut self.attributions[pending_start..] {
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
attribution.anchor =
PendingAnchor::Resolved(TokenAnchor::ZeroWidth { byte_offset });
}
}
}
/// Truncate text and collapse removed anchors to the new boundary.
pub(super) fn truncate(&mut self, byte_offset: usize) {
self.text.truncate(byte_offset);
let byte_offset = offset_as_u32(byte_offset);
for attribution in &mut self.attributions {
let PendingAnchor::Resolved(anchor) = &mut attribution.anchor else {
continue;
};
if anchor.byte_offset() >= byte_offset {
*anchor = TokenAnchor::ZeroWidth { byte_offset };
}
}
}
/// Emit decoded text and attributions ready through `cutoff`.
pub(super) fn take_ready(&mut self, cutoff: usize) -> Option<DecodedText> {
let chunk_start = self.emitted_byte_offset;
let cutoff_u32 = offset_as_u32(cutoff);
let mut attribution_end = self.emitted_attribution_count;
for attribution in &self.attributions[self.emitted_attribution_count..] {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
break;
};
let ready = match anchor {
TokenAnchor::Visible { byte_offset } => byte_offset < cutoff_u32,
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= cutoff_u32,
};
if !ready {
break;
}
attribution_end += 1;
}
if cutoff == chunk_start && attribution_end == self.emitted_attribution_count {
return None;
}
let chunk_start_u32 = offset_as_u32(chunk_start);
let attributions = self.attributions[self.emitted_attribution_count..attribution_end]
.iter()
.map(|attribution| {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
unreachable!("ready attributions must be resolved")
};
TokenAttribution {
token_id: attribution.token_id,
anchor: anchor.offset_by(-i64::from(chunk_start_u32)),
}
})
.collect();
let text = self.text[chunk_start..cutoff].to_string();
self.emitted_byte_offset = cutoff;
self.emitted_attribution_count = attribution_end;
Some(DecodedText { text, attributions })
}
/// Emit the final chunk and take the complete decoded output.
pub(super) fn finish(&mut self) -> (Option<DecodedText>, DecodedText) {
let last_chunk = self.take_ready(self.text.len());
let attributions = take(&mut self.attributions)
.into_iter()
.map(|attribution| {
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
unreachable!("finish requires every attribution to be resolved")
};
TokenAttribution {
token_id: attribution.token_id,
anchor,
}
})
.collect();
let full = DecodedText {
text: take(&mut self.text),
attributions,
};
self.emitted_byte_offset = 0;
self.emitted_attribution_count = 0;
self.pending_start = None;
(last_chunk, full)
}
}
fn offset_as_u32(offset: usize) -> u32 {
u32::try_from(offset).expect("decoded text exceeds 4 GiB")
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,425 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use super::*;
use crate::{Result, Tokenizer, test_utils::TestTokenizer};
const SPECIAL_TOKEN_ID: u32 = 0x100;
/// Backend with tokens that decode to whole text pieces.
#[derive(Debug)]
struct PieceBackend;
impl Tokenizer for PieceBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
Ok(token_ids
.iter()
.map(|token_id| match token_id {
1 => "abcd",
2 => "ab",
3 => "<stop>",
4 => "",
_ => unreachable!("unexpected token id: {token_id}"),
})
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
}
/// Raw-byte backend with one filterable special token.
#[derive(Debug)]
struct Utf8SpecialBackend;
impl Tokenizer for Utf8SpecialBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let mut bytes = Vec::new();
for &token_id in token_ids {
if token_id == SPECIAL_TOKEN_ID {
if !skip_special_tokens {
bytes.extend_from_slice(b"<special>");
}
} else {
bytes.push(token_id as u8);
}
}
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == SPECIAL_TOKEN_ID
}
}
fn tokenizer_with_special_token() -> TestTokenizer {
TestTokenizer::new().with_special_token("<special>", SPECIAL_TOKEN_ID)
}
fn visible(byte_offset: u32) -> TokenAnchor {
TokenAnchor::Visible { byte_offset }
}
fn zero_width(byte_offset: u32) -> TokenAnchor {
TokenAnchor::ZeroWidth { byte_offset }
}
fn decoded(text: &str, token_ids: &[u32], anchors: &[TokenAnchor]) -> DecodedText {
assert_eq!(token_ids.len(), anchors.len());
DecodedText {
text: text.to_string(),
attributions: token_ids
.iter()
.copied()
.zip(anchors.iter().copied())
.map(|(token_id, anchor)| TokenAttribution { token_id, anchor })
.collect(),
}
}
#[test]
fn token_anchor_offset_by_preserves_anchor_kind() {
for (anchor, delta, expected) in [
(visible(3), -2, visible(1)),
(zero_width(1), 4, zero_width(5)),
] {
assert_eq!(anchor.offset_by(delta), expected);
}
}
#[test]
fn decoded_text_append_rebases_attributions() {
let mut combined = decoded("a", &[1], &[visible(0)]);
combined.append(decoded(
"你",
&[2, 3, 4, 5],
&[visible(0), visible(0), visible(0), zero_width(3)],
));
assert_eq!(
combined,
decoded(
"a你",
&[1, 2, 3, 4, 5],
&[
visible(0),
visible(1),
visible(1),
visible(1),
zero_width(4),
],
)
);
}
#[test]
fn decoded_text_append_moves_allocations_into_empty_receiver() {
let source = decoded(
"hello",
&[1, 2, 3, 4, 5],
&[visible(0), visible(1), visible(2), visible(3), visible(4)],
);
let text_ptr = source.text.as_ptr();
let attributions_ptr = source.attributions.as_ptr();
let mut combined = DecodedText::default();
combined.append(source);
assert_eq!(combined.text.as_ptr(), text_ptr);
assert_eq!(combined.attributions.as_ptr(), attributions_ptr);
}
#[test]
fn decoded_text_append_preserves_zero_width_only_receiver() {
let mut combined = decoded("", &[1], &[zero_width(0)]);
combined.append(decoded("a", &[2], &[visible(0)]));
assert_eq!(
combined,
decoded("a", &[1, 2], &[zero_width(0), visible(0)])
);
}
struct AttributionCase<'a> {
tokenizer: &'a dyn Tokenizer,
prompt_token_ids: Vec<u32>,
token_ids: Vec<u32>,
skip_special_tokens: bool,
min_bytes_to_buffer: usize,
truncate_output_to: Option<usize>,
expected_chunks: Vec<DecodedText>,
expected_full: DecodedText,
}
fn run_attribution_case(case: AttributionCase<'_>) {
let mut decoder = case.tokenizer.create_decode_stream(
&case.prompt_token_ids,
case.skip_special_tokens,
case.min_bytes_to_buffer,
);
let mut chunks = Vec::new();
for token_id in &case.token_ids {
decoder.push_token(*token_id).unwrap();
while let Some(chunk) = decoder.next_chunk() {
chunks.push(chunk);
}
}
let (remaining, full) = decoder.flush(case.truncate_output_to).unwrap();
if let Some(remaining) = remaining {
chunks.push(remaining);
}
assert_eq!(chunks, case.expected_chunks, "chunks");
assert_eq!(full, case.expected_full, "full output");
assert_eq!(
full.attributions.len(),
case.token_ids.len(),
"one attribution per pushed token"
);
assert_eq!(
full.attributions
.iter()
.map(|attribution| attribution.token_id)
.collect::<Vec<_>>(),
case.token_ids,
"attribution preserves pushed token IDs"
);
let mut reconstructed_text = String::new();
let mut reconstructed_attributions = Vec::new();
for chunk in &chunks {
let chunk_start = reconstructed_text.len() as u32;
reconstructed_text.push_str(&chunk.text);
reconstructed_attributions.extend(chunk.attributions.iter().map(|attribution| {
TokenAttribution {
token_id: attribution.token_id,
anchor: attribution.anchor.offset_by(i64::from(chunk_start)),
}
}));
}
assert_eq!(
reconstructed_text, full.text,
"chunk text reconstructs full text"
);
assert_eq!(
reconstructed_attributions.as_slice(),
full.attributions.as_slice(),
"chunk attributions reconstruct full attributions"
);
}
#[test]
fn attribution_ascii_tokens_have_one_visible_anchor_each() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![b'o' as u32, b'k' as u32],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![
decoded("o", &[b'o' as u32], &[visible(0)]),
decoded("k", &[b'k' as u32], &[visible(0)]),
],
expected_full: decoded("ok", &[b'o' as u32, b'k' as u32], &[visible(0), visible(1)]),
});
}
#[test]
fn attribution_byte_fallback_tokens_share_one_visible_anchor() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![0xe4, 0xbd, 0xa0],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded(
"你",
&[0xe4, 0xbd, 0xa0],
&[visible(0), visible(0), visible(0)],
)],
expected_full: decoded(
"你",
&[0xe4, 0xbd, 0xa0],
&[visible(0), visible(0), visible(0)],
),
});
}
#[test]
fn attribution_holdback_splits_text_without_repeating_token() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![1],
skip_special_tokens: false,
min_bytes_to_buffer: 2,
truncate_output_to: None,
expected_chunks: vec![decoded("ab", &[1], &[visible(0)]), decoded("cd", &[], &[])],
expected_full: decoded("abcd", &[1], &[visible(0)]),
});
}
#[test]
fn attribution_filtered_special_token_is_zero_width_at_its_byte_boundary() {
let tokenizer = tokenizer_with_special_token();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
skip_special_tokens: true,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![
decoded("a", &[b'a' as u32], &[visible(0)]),
decoded("", &[SPECIAL_TOKEN_ID], &[zero_width(0)]),
decoded("b", &[b'b' as u32], &[visible(0)]),
],
expected_full: decoded(
"ab",
&[b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
&[visible(0), zero_width(1), visible(1)],
),
});
}
#[test]
fn attribution_zero_width_preserves_order_inside_byte_fallback_group() {
run_attribution_case(AttributionCase {
tokenizer: &Utf8SpecialBackend,
prompt_token_ids: vec![],
token_ids: vec![0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
skip_special_tokens: true,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded(
"你",
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
&[visible(0), zero_width(0), visible(0), visible(0)],
)],
expected_full: decoded(
"你",
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
&[visible(0), zero_width(0), visible(0), visible(0)],
),
});
}
#[test]
fn attribution_retained_special_token_has_visible_anchor() {
run_attribution_case(AttributionCase {
tokenizer: &Utf8SpecialBackend,
prompt_token_ids: vec![],
token_ids: vec![SPECIAL_TOKEN_ID],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)])],
expected_full: decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)]),
});
}
#[test]
fn attribution_empty_decode_resolves_to_zero_width_on_flush() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![4],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("", &[4], &[zero_width(0)])],
expected_full: decoded("", &[4], &[zero_width(0)]),
});
}
#[test]
fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![],
token_ids: vec![0xe4, 0xbd],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)])],
expected_full: decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)]),
});
}
#[test]
fn attribution_truncation_converts_fully_removed_token_to_zero_width() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![2, 3],
skip_special_tokens: false,
min_bytes_to_buffer: 32,
truncate_output_to: Some(2),
expected_chunks: vec![decoded("ab", &[2, 3], &[visible(0), zero_width(2)])],
expected_full: decoded("ab", &[2, 3], &[visible(0), zero_width(2)]),
});
}
#[test]
fn attribution_truncation_inside_token_retains_first_byte_anchor() {
run_attribution_case(AttributionCase {
tokenizer: &PieceBackend,
prompt_token_ids: vec![],
token_ids: vec![1],
skip_special_tokens: false,
min_bytes_to_buffer: 32,
truncate_output_to: Some(2),
expected_chunks: vec![decoded("ab", &[1], &[visible(0)])],
expected_full: decoded("ab", &[1], &[visible(0)]),
});
}
#[test]
fn attribution_excludes_prompt_context() {
let tokenizer = TestTokenizer::new();
run_attribution_case(AttributionCase {
tokenizer: &tokenizer,
prompt_token_ids: vec![b'H' as u32, b'i' as u32],
token_ids: vec![b'!' as u32],
skip_special_tokens: false,
min_bytes_to_buffer: 0,
truncate_output_to: None,
expected_chunks: vec![decoded("!", &[b'!' as u32], &[visible(0)])],
expected_full: decoded("!", &[b'!' as u32], &[visible(0)]),
});
}
+1 -1
View File
@@ -17,7 +17,7 @@ mod tiktoken;
pub use error::{Result, TokenizerError};
pub use hf::HuggingFaceTokenizer;
pub use incremental::IncrementalDecoder;
pub use incremental::{DecodedText, IncrementalDecoder, TokenAnchor, TokenAttribution};
pub use tekken::TekkenTokenizer;
pub use tiktoken::TiktokenTokenizer;
+8 -8
View File
@@ -964,16 +964,16 @@ mod tests {
for &id in &ids {
decoder.push_token(id).unwrap();
if let Some(chunk) = decoder.next_chunk() {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
assert_eq!(output, text);
assert_eq!(full_text, text);
assert_eq!(full.text, text);
}
}
@@ -991,16 +991,16 @@ mod tests {
for &id in &ids {
decoder.push_token(id).unwrap();
if let Some(chunk) = decoder.next_chunk() {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
let (last_chunk, full) = decoder.flush(None).unwrap();
if let Some(chunk) = last_chunk {
output.push_str(&chunk);
output.push_str(&chunk.text);
}
assert_eq!(output, text);
assert_eq!(full_text, text);
assert_eq!(full.text, text);
}
}