[Rust Frontend] Attribute decoded text to tokens (#52910)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -14,6 +14,7 @@ riptoken.workspace = true
|
||||
rustc-hash.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
smallvec.workspace = true
|
||||
tekken.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
use crate::{Result, Tokenizer};
|
||||
|
||||
mod attribution;
|
||||
|
||||
use attribution::AttributedTextBuffer;
|
||||
pub use attribution::{DecodedText, TokenAnchor, TokenAttribution};
|
||||
|
||||
/// Stateful incremental decoder that emits text chunks one token at a time.
|
||||
pub trait IncrementalDecoder: Send {
|
||||
/// Push one generated token and return how many new string bytes were
|
||||
@@ -12,13 +15,16 @@ pub trait IncrementalDecoder: Send {
|
||||
fn push_token(&mut self, token_id: u32) -> Result<usize>;
|
||||
|
||||
/// Consume any text which is currently ready.
|
||||
fn next_chunk(&mut self) -> Option<String>;
|
||||
fn next_chunk(&mut self) -> Option<DecodedText>;
|
||||
|
||||
/// Flush any remaining buffered text that has not yet been emitted.
|
||||
///
|
||||
/// Called after the final generated token to force out buffered/incomplete
|
||||
/// fragments.
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
|
||||
fn flush(
|
||||
&mut self,
|
||||
truncate_output_to: Option<usize>,
|
||||
) -> Result<(Option<DecodedText>, DecodedText)>;
|
||||
|
||||
/// Return cumulative decoded text so far.
|
||||
fn output(&self) -> &str;
|
||||
@@ -36,8 +42,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
|
||||
prefix: String,
|
||||
prefix_index: usize,
|
||||
prefix_seeded: bool,
|
||||
cumulative_output: String,
|
||||
output_index: usize,
|
||||
decoded: AttributedTextBuffer,
|
||||
}
|
||||
|
||||
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
|
||||
@@ -55,8 +60,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
|
||||
prefix: String::new(),
|
||||
prefix_index: 0,
|
||||
prefix_seeded: prompt_token_ids.is_empty(),
|
||||
cumulative_output: String::new(),
|
||||
output_index: 0,
|
||||
decoded: AttributedTextBuffer::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,6 +72,14 @@ const SAFE_SUFFIX_MIN: usize = 4;
|
||||
const SAFE_SUFFIX_MAX: usize = 6;
|
||||
|
||||
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
|
||||
fn record_token(&mut self, token_id: u32) {
|
||||
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
|
||||
self.decoded.record_zero_width_token(token_id);
|
||||
} else {
|
||||
self.decoded.record_pending_token(token_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode prompt-only context for prefix seeding.
|
||||
///
|
||||
/// Prompt ids may come from the model vocabulary rather than the local
|
||||
@@ -132,6 +144,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
}
|
||||
|
||||
self.ids.push(token_id);
|
||||
self.record_token(token_id);
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
|
||||
@@ -139,49 +152,48 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
}
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
|
||||
self.cumulative_output.push_str(new_chunk);
|
||||
self.decoded.append_visible_text(new_chunk);
|
||||
self.ids.drain(..self.prefix_index);
|
||||
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
self.prefix_index = self.ids.len();
|
||||
Ok(new_chunk.len())
|
||||
}
|
||||
|
||||
fn next_chunk(&mut self) -> Option<String> {
|
||||
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
|
||||
fn next_chunk(&mut self) -> Option<DecodedText> {
|
||||
let cutoff = self.decoded.len().saturating_sub(self.min_bytes_to_buffer);
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
let cutoff = self.cumulative_output.floor_char_boundary(cutoff);
|
||||
(cutoff > self.output_index).then(|| {
|
||||
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
|
||||
self.output_index = cutoff;
|
||||
chunk
|
||||
})
|
||||
let cutoff = self.decoded.text().floor_char_boundary(cutoff);
|
||||
self.decoded.take_ready(cutoff)
|
||||
}
|
||||
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
|
||||
fn flush(
|
||||
&mut self,
|
||||
truncate_output_to: Option<usize>,
|
||||
) -> Result<(Option<DecodedText>, DecodedText)> {
|
||||
// If the prefix was never seeded (no push_token was called), `ids`
|
||||
// holds only prompt context — decoding it would re-emit prompt text.
|
||||
if self.prefix_seeded && !self.ids.is_empty() {
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
self.cumulative_output
|
||||
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
|
||||
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
|
||||
if !new_chunk.is_empty() {
|
||||
self.decoded.append_visible_text(new_chunk);
|
||||
}
|
||||
}
|
||||
self.decoded.resolve_pending_zero_width();
|
||||
self.ids.clear();
|
||||
self.prefix.clear();
|
||||
self.prefix_index = 0;
|
||||
self.prefix_seeded = true;
|
||||
if let Some(truncate_output_to) = truncate_output_to {
|
||||
self.cumulative_output.truncate(truncate_output_to);
|
||||
self.decoded.truncate(truncate_output_to);
|
||||
}
|
||||
let last_chunk = (self.output_index < self.cumulative_output.len())
|
||||
.then(|| self.cumulative_output[self.output_index..].to_string());
|
||||
self.output_index = 0;
|
||||
Ok((last_chunk, take(&mut self.cumulative_output)))
|
||||
Ok(self.decoded.finish())
|
||||
}
|
||||
|
||||
fn output(&self) -> &str {
|
||||
&self.cumulative_output
|
||||
self.decoded.text()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,13 +257,13 @@ mod tests {
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
|
||||
assert_eq!(decoder.next_chunk().unwrap().text, "o");
|
||||
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
|
||||
assert_eq!(decoder.next_chunk().unwrap().text, "k");
|
||||
// All text already consumed via next_chunk
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "ok");
|
||||
assert_eq!(full_text.text, "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -288,6 +300,7 @@ mod tests {
|
||||
0 if !skip_special_tokens => text.push_str("<special>"),
|
||||
0 => {}
|
||||
1 => text.push('a'),
|
||||
2 => text.push('b'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -301,6 +314,10 @@ mod tests {
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -393,17 +410,17 @@ mod tests {
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
let input = b"Hello, world!";
|
||||
let mut full = String::new();
|
||||
let mut out = String::new();
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
full.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None); // all consumed via next_chunk
|
||||
assert_eq!(full, "Hello, world!");
|
||||
assert_eq!(full_text, "Hello, world!");
|
||||
assert_eq!(out, "Hello, world!");
|
||||
assert_eq!(full.text, "Hello, world!");
|
||||
}
|
||||
|
||||
/// Backend simulating non-monotonic decode where adding a token changes how
|
||||
@@ -470,15 +487,15 @@ mod tests {
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
chunks.push_str(&chunk);
|
||||
chunks.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
|
||||
assert_eq!(chunks, "Hel");
|
||||
// Flush returns the rest
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk.as_deref(), Some("lo!"));
|
||||
assert_eq!(full_text, "Hello!");
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk.unwrap().text, "lo!");
|
||||
assert_eq!(full.text, "Hello!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -493,14 +510,14 @@ mod tests {
|
||||
for byte in "你好A".bytes() {
|
||||
decoder.push_token(u32::from(byte)).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
out.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
out.push_str(&chunk);
|
||||
out.push_str(&chunk.text);
|
||||
}
|
||||
assert_eq!(full_text, "你好A");
|
||||
assert_eq!(full.text, "你好A");
|
||||
assert_eq!(out, "你好A");
|
||||
}
|
||||
|
||||
@@ -515,9 +532,9 @@ mod tests {
|
||||
.collect();
|
||||
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
|
||||
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "");
|
||||
assert_eq!(full.text, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -526,8 +543,8 @@ mod tests {
|
||||
let prompt = vec![0xe4, 0xbd];
|
||||
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
|
||||
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "");
|
||||
assert_eq!(full.text, "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::mem::take;
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
/// Position of one generated token in decoded text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TokenAnchor {
|
||||
/// The token contributed to visible decoded text.
|
||||
Visible { byte_offset: u32 },
|
||||
/// The token produced no visible decoded bytes.
|
||||
ZeroWidth { byte_offset: u32 },
|
||||
}
|
||||
|
||||
impl TokenAnchor {
|
||||
fn byte_offset(self) -> u32 {
|
||||
match self {
|
||||
Self::Visible { byte_offset } | Self::ZeroWidth { byte_offset } => byte_offset,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a new anchor offset by `delta` bytes.
|
||||
#[must_use]
|
||||
pub(super) fn offset_by(self, delta: i64) -> Self {
|
||||
let byte_offset = i64::from(self.byte_offset())
|
||||
.checked_add(delta)
|
||||
.and_then(|offset| u32::try_from(offset).ok())
|
||||
.expect("token anchor byte offset out of range");
|
||||
match self {
|
||||
Self::Visible { .. } => Self::Visible { byte_offset },
|
||||
Self::ZeroWidth { .. } => Self::ZeroWidth { byte_offset },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One generated token and its position in decoded text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TokenAttribution {
|
||||
/// Token ID passed to [`super::IncrementalDecoder::push_token`].
|
||||
pub token_id: u32,
|
||||
/// Position of the token in the decoded text.
|
||||
pub anchor: TokenAnchor,
|
||||
}
|
||||
|
||||
/// Decoded text and the generated tokens attributed to it.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct DecodedText {
|
||||
/// Decoded UTF-8 text.
|
||||
pub text: String,
|
||||
/// One entry per generated token, in generation order.
|
||||
///
|
||||
/// Offsets are local to `text`. Visible offsets may repeat when multiple
|
||||
/// tokens jointly decode into one character. A zero-width offset may equal
|
||||
/// `text.len()`.
|
||||
/// Four inline records cover one complete UTF-8 byte-fallback sequence.
|
||||
pub attributions: SmallVec<[TokenAttribution; 4]>,
|
||||
}
|
||||
|
||||
impl DecodedText {
|
||||
/// Create decoded text without token attribution records. Mostly for testing purposes.
|
||||
pub fn unattributed(text: impl Into<String>) -> Self {
|
||||
Self {
|
||||
text: text.into(),
|
||||
attributions: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append another decoded fragment, rebasing its token anchors.
|
||||
pub fn append(&mut self, other: Self) {
|
||||
if self.text.is_empty() && self.attributions.is_empty() {
|
||||
*self = other;
|
||||
return;
|
||||
}
|
||||
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
let other_len = offset_as_u32(other.text.len());
|
||||
let _combined_len = byte_offset.checked_add(other_len).expect("decoded text exceeds 4 GiB");
|
||||
|
||||
self.text.push_str(&other.text);
|
||||
self.attributions.extend(other.attributions.into_iter().map(|attribution| {
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: attribution.anchor.offset_by(i64::from(byte_offset)),
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum PendingAnchor {
|
||||
Unresolved,
|
||||
Resolved(TokenAnchor),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct PendingAttribution {
|
||||
token_id: u32,
|
||||
anchor: PendingAnchor,
|
||||
}
|
||||
|
||||
/// Cumulative decoded text plus the attribution state needed to emit local
|
||||
/// chunks.
|
||||
#[derive(Default)]
|
||||
pub(super) struct AttributedTextBuffer {
|
||||
text: String,
|
||||
emitted_byte_offset: usize,
|
||||
attributions: Vec<PendingAttribution>,
|
||||
emitted_attribution_count: usize,
|
||||
pending_start: Option<usize>,
|
||||
}
|
||||
|
||||
impl AttributedTextBuffer {
|
||||
/// Return all decoded text accumulated so far.
|
||||
pub(super) fn text(&self) -> &str {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Return the decoded text length in bytes.
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.text.len()
|
||||
}
|
||||
|
||||
/// Record a token awaiting visible decoded output.
|
||||
pub(super) fn record_pending_token(&mut self, token_id: u32) {
|
||||
self.pending_start.get_or_insert(self.attributions.len());
|
||||
self.attributions.push(PendingAttribution {
|
||||
token_id,
|
||||
anchor: PendingAnchor::Unresolved,
|
||||
});
|
||||
}
|
||||
|
||||
/// Record a token that produced no visible decoded output.
|
||||
pub(super) fn record_zero_width_token(&mut self, token_id: u32) {
|
||||
self.attributions.push(PendingAttribution {
|
||||
token_id,
|
||||
anchor: PendingAnchor::Resolved(TokenAnchor::ZeroWidth {
|
||||
byte_offset: offset_as_u32(self.text.len()),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/// Append visible text and anchor pending tokens at its start.
|
||||
pub(super) fn append_visible_text(&mut self, text: &str) {
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
if let Some(pending_start) = self.pending_start.take() {
|
||||
for attribution in &mut self.attributions[pending_start..] {
|
||||
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
|
||||
attribution.anchor =
|
||||
PendingAnchor::Resolved(TokenAnchor::Visible { byte_offset });
|
||||
}
|
||||
}
|
||||
}
|
||||
self.text.push_str(text);
|
||||
}
|
||||
|
||||
/// Resolve remaining pending tokens at the current text boundary.
|
||||
pub(super) fn resolve_pending_zero_width(&mut self) {
|
||||
let Some(pending_start) = self.pending_start.take() else {
|
||||
return;
|
||||
};
|
||||
let byte_offset = offset_as_u32(self.text.len());
|
||||
for attribution in &mut self.attributions[pending_start..] {
|
||||
if matches!(attribution.anchor, PendingAnchor::Unresolved) {
|
||||
attribution.anchor =
|
||||
PendingAnchor::Resolved(TokenAnchor::ZeroWidth { byte_offset });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate text and collapse removed anchors to the new boundary.
|
||||
pub(super) fn truncate(&mut self, byte_offset: usize) {
|
||||
self.text.truncate(byte_offset);
|
||||
let byte_offset = offset_as_u32(byte_offset);
|
||||
for attribution in &mut self.attributions {
|
||||
let PendingAnchor::Resolved(anchor) = &mut attribution.anchor else {
|
||||
continue;
|
||||
};
|
||||
if anchor.byte_offset() >= byte_offset {
|
||||
*anchor = TokenAnchor::ZeroWidth { byte_offset };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit decoded text and attributions ready through `cutoff`.
|
||||
pub(super) fn take_ready(&mut self, cutoff: usize) -> Option<DecodedText> {
|
||||
let chunk_start = self.emitted_byte_offset;
|
||||
let cutoff_u32 = offset_as_u32(cutoff);
|
||||
let mut attribution_end = self.emitted_attribution_count;
|
||||
|
||||
for attribution in &self.attributions[self.emitted_attribution_count..] {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
break;
|
||||
};
|
||||
let ready = match anchor {
|
||||
TokenAnchor::Visible { byte_offset } => byte_offset < cutoff_u32,
|
||||
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= cutoff_u32,
|
||||
};
|
||||
if !ready {
|
||||
break;
|
||||
}
|
||||
attribution_end += 1;
|
||||
}
|
||||
|
||||
if cutoff == chunk_start && attribution_end == self.emitted_attribution_count {
|
||||
return None;
|
||||
}
|
||||
|
||||
let chunk_start_u32 = offset_as_u32(chunk_start);
|
||||
let attributions = self.attributions[self.emitted_attribution_count..attribution_end]
|
||||
.iter()
|
||||
.map(|attribution| {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
unreachable!("ready attributions must be resolved")
|
||||
};
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: anchor.offset_by(-i64::from(chunk_start_u32)),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let text = self.text[chunk_start..cutoff].to_string();
|
||||
|
||||
self.emitted_byte_offset = cutoff;
|
||||
self.emitted_attribution_count = attribution_end;
|
||||
Some(DecodedText { text, attributions })
|
||||
}
|
||||
|
||||
/// Emit the final chunk and take the complete decoded output.
|
||||
pub(super) fn finish(&mut self) -> (Option<DecodedText>, DecodedText) {
|
||||
let last_chunk = self.take_ready(self.text.len());
|
||||
let attributions = take(&mut self.attributions)
|
||||
.into_iter()
|
||||
.map(|attribution| {
|
||||
let PendingAnchor::Resolved(anchor) = attribution.anchor else {
|
||||
unreachable!("finish requires every attribution to be resolved")
|
||||
};
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let full = DecodedText {
|
||||
text: take(&mut self.text),
|
||||
attributions,
|
||||
};
|
||||
self.emitted_byte_offset = 0;
|
||||
self.emitted_attribution_count = 0;
|
||||
self.pending_start = None;
|
||||
(last_chunk, full)
|
||||
}
|
||||
}
|
||||
|
||||
fn offset_as_u32(offset: usize) -> u32 {
|
||||
u32::try_from(offset).expect("decoded text exceeds 4 GiB")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,425 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use super::*;
|
||||
use crate::{Result, Tokenizer, test_utils::TestTokenizer};
|
||||
|
||||
const SPECIAL_TOKEN_ID: u32 = 0x100;
|
||||
|
||||
/// Backend with tokens that decode to whole text pieces.
|
||||
#[derive(Debug)]
|
||||
struct PieceBackend;
|
||||
|
||||
impl Tokenizer for PieceBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| match token_id {
|
||||
1 => "abcd",
|
||||
2 => "ab",
|
||||
3 => "<stop>",
|
||||
4 => "",
|
||||
_ => unreachable!("unexpected token id: {token_id}"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw-byte backend with one filterable special token.
|
||||
#[derive(Debug)]
|
||||
struct Utf8SpecialBackend;
|
||||
|
||||
impl Tokenizer for Utf8SpecialBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let mut bytes = Vec::new();
|
||||
for &token_id in token_ids {
|
||||
if token_id == SPECIAL_TOKEN_ID {
|
||||
if !skip_special_tokens {
|
||||
bytes.extend_from_slice(b"<special>");
|
||||
}
|
||||
} else {
|
||||
bytes.push(token_id as u8);
|
||||
}
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == SPECIAL_TOKEN_ID
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer_with_special_token() -> TestTokenizer {
|
||||
TestTokenizer::new().with_special_token("<special>", SPECIAL_TOKEN_ID)
|
||||
}
|
||||
|
||||
fn visible(byte_offset: u32) -> TokenAnchor {
|
||||
TokenAnchor::Visible { byte_offset }
|
||||
}
|
||||
|
||||
fn zero_width(byte_offset: u32) -> TokenAnchor {
|
||||
TokenAnchor::ZeroWidth { byte_offset }
|
||||
}
|
||||
|
||||
fn decoded(text: &str, token_ids: &[u32], anchors: &[TokenAnchor]) -> DecodedText {
|
||||
assert_eq!(token_ids.len(), anchors.len());
|
||||
DecodedText {
|
||||
text: text.to_string(),
|
||||
attributions: token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.zip(anchors.iter().copied())
|
||||
.map(|(token_id, anchor)| TokenAttribution { token_id, anchor })
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_anchor_offset_by_preserves_anchor_kind() {
|
||||
for (anchor, delta, expected) in [
|
||||
(visible(3), -2, visible(1)),
|
||||
(zero_width(1), 4, zero_width(5)),
|
||||
] {
|
||||
assert_eq!(anchor.offset_by(delta), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_rebases_attributions() {
|
||||
let mut combined = decoded("a", &[1], &[visible(0)]);
|
||||
combined.append(decoded(
|
||||
"你",
|
||||
&[2, 3, 4, 5],
|
||||
&[visible(0), visible(0), visible(0), zero_width(3)],
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
combined,
|
||||
decoded(
|
||||
"a你",
|
||||
&[1, 2, 3, 4, 5],
|
||||
&[
|
||||
visible(0),
|
||||
visible(1),
|
||||
visible(1),
|
||||
visible(1),
|
||||
zero_width(4),
|
||||
],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_moves_allocations_into_empty_receiver() {
|
||||
let source = decoded(
|
||||
"hello",
|
||||
&[1, 2, 3, 4, 5],
|
||||
&[visible(0), visible(1), visible(2), visible(3), visible(4)],
|
||||
);
|
||||
let text_ptr = source.text.as_ptr();
|
||||
let attributions_ptr = source.attributions.as_ptr();
|
||||
let mut combined = DecodedText::default();
|
||||
|
||||
combined.append(source);
|
||||
|
||||
assert_eq!(combined.text.as_ptr(), text_ptr);
|
||||
assert_eq!(combined.attributions.as_ptr(), attributions_ptr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_append_preserves_zero_width_only_receiver() {
|
||||
let mut combined = decoded("", &[1], &[zero_width(0)]);
|
||||
combined.append(decoded("a", &[2], &[visible(0)]));
|
||||
|
||||
assert_eq!(
|
||||
combined,
|
||||
decoded("a", &[1, 2], &[zero_width(0), visible(0)])
|
||||
);
|
||||
}
|
||||
|
||||
struct AttributionCase<'a> {
|
||||
tokenizer: &'a dyn Tokenizer,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
token_ids: Vec<u32>,
|
||||
skip_special_tokens: bool,
|
||||
min_bytes_to_buffer: usize,
|
||||
truncate_output_to: Option<usize>,
|
||||
expected_chunks: Vec<DecodedText>,
|
||||
expected_full: DecodedText,
|
||||
}
|
||||
|
||||
fn run_attribution_case(case: AttributionCase<'_>) {
|
||||
let mut decoder = case.tokenizer.create_decode_stream(
|
||||
&case.prompt_token_ids,
|
||||
case.skip_special_tokens,
|
||||
case.min_bytes_to_buffer,
|
||||
);
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
for token_id in &case.token_ids {
|
||||
decoder.push_token(*token_id).unwrap();
|
||||
while let Some(chunk) = decoder.next_chunk() {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
let (remaining, full) = decoder.flush(case.truncate_output_to).unwrap();
|
||||
if let Some(remaining) = remaining {
|
||||
chunks.push(remaining);
|
||||
}
|
||||
|
||||
assert_eq!(chunks, case.expected_chunks, "chunks");
|
||||
assert_eq!(full, case.expected_full, "full output");
|
||||
assert_eq!(
|
||||
full.attributions.len(),
|
||||
case.token_ids.len(),
|
||||
"one attribution per pushed token"
|
||||
);
|
||||
assert_eq!(
|
||||
full.attributions
|
||||
.iter()
|
||||
.map(|attribution| attribution.token_id)
|
||||
.collect::<Vec<_>>(),
|
||||
case.token_ids,
|
||||
"attribution preserves pushed token IDs"
|
||||
);
|
||||
|
||||
let mut reconstructed_text = String::new();
|
||||
let mut reconstructed_attributions = Vec::new();
|
||||
for chunk in &chunks {
|
||||
let chunk_start = reconstructed_text.len() as u32;
|
||||
reconstructed_text.push_str(&chunk.text);
|
||||
reconstructed_attributions.extend(chunk.attributions.iter().map(|attribution| {
|
||||
TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: attribution.anchor.offset_by(i64::from(chunk_start)),
|
||||
}
|
||||
}));
|
||||
}
|
||||
assert_eq!(
|
||||
reconstructed_text, full.text,
|
||||
"chunk text reconstructs full text"
|
||||
);
|
||||
assert_eq!(
|
||||
reconstructed_attributions.as_slice(),
|
||||
full.attributions.as_slice(),
|
||||
"chunk attributions reconstruct full attributions"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_ascii_tokens_have_one_visible_anchor_each() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![b'o' as u32, b'k' as u32],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![
|
||||
decoded("o", &[b'o' as u32], &[visible(0)]),
|
||||
decoded("k", &[b'k' as u32], &[visible(0)]),
|
||||
],
|
||||
expected_full: decoded("ok", &[b'o' as u32, b'k' as u32], &[visible(0), visible(1)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_byte_fallback_tokens_share_one_visible_anchor() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, 0xbd, 0xa0],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded(
|
||||
"你",
|
||||
&[0xe4, 0xbd, 0xa0],
|
||||
&[visible(0), visible(0), visible(0)],
|
||||
)],
|
||||
expected_full: decoded(
|
||||
"你",
|
||||
&[0xe4, 0xbd, 0xa0],
|
||||
&[visible(0), visible(0), visible(0)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_holdback_splits_text_without_repeating_token() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![1],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 2,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("ab", &[1], &[visible(0)]), decoded("cd", &[], &[])],
|
||||
expected_full: decoded("abcd", &[1], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_filtered_special_token_is_zero_width_at_its_byte_boundary() {
|
||||
let tokenizer = tokenizer_with_special_token();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
|
||||
skip_special_tokens: true,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![
|
||||
decoded("a", &[b'a' as u32], &[visible(0)]),
|
||||
decoded("", &[SPECIAL_TOKEN_ID], &[zero_width(0)]),
|
||||
decoded("b", &[b'b' as u32], &[visible(0)]),
|
||||
],
|
||||
expected_full: decoded(
|
||||
"ab",
|
||||
&[b'a' as u32, SPECIAL_TOKEN_ID, b'b' as u32],
|
||||
&[visible(0), zero_width(1), visible(1)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_zero_width_preserves_order_inside_byte_fallback_group() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &Utf8SpecialBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
skip_special_tokens: true,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded(
|
||||
"你",
|
||||
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
&[visible(0), zero_width(0), visible(0), visible(0)],
|
||||
)],
|
||||
expected_full: decoded(
|
||||
"你",
|
||||
&[0xe4, SPECIAL_TOKEN_ID, 0xbd, 0xa0],
|
||||
&[visible(0), zero_width(0), visible(0), visible(0)],
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_retained_special_token_has_visible_anchor() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &Utf8SpecialBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![SPECIAL_TOKEN_ID],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)])],
|
||||
expected_full: decoded("<special>", &[SPECIAL_TOKEN_ID], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_empty_decode_resolves_to_zero_width_on_flush() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![4],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("", &[4], &[zero_width(0)])],
|
||||
expected_full: decoded("", &[4], &[zero_width(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![0xe4, 0xbd],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)])],
|
||||
expected_full: decoded("�", &[0xe4, 0xbd], &[visible(0), visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_truncation_converts_fully_removed_token_to_zero_width() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![2, 3],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 32,
|
||||
truncate_output_to: Some(2),
|
||||
expected_chunks: vec![decoded("ab", &[2, 3], &[visible(0), zero_width(2)])],
|
||||
expected_full: decoded("ab", &[2, 3], &[visible(0), zero_width(2)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_truncation_inside_token_retains_first_byte_anchor() {
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &PieceBackend,
|
||||
prompt_token_ids: vec![],
|
||||
token_ids: vec![1],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 32,
|
||||
truncate_output_to: Some(2),
|
||||
expected_chunks: vec![decoded("ab", &[1], &[visible(0)])],
|
||||
expected_full: decoded("ab", &[1], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attribution_excludes_prompt_context() {
|
||||
let tokenizer = TestTokenizer::new();
|
||||
run_attribution_case(AttributionCase {
|
||||
tokenizer: &tokenizer,
|
||||
prompt_token_ids: vec![b'H' as u32, b'i' as u32],
|
||||
token_ids: vec![b'!' as u32],
|
||||
skip_special_tokens: false,
|
||||
min_bytes_to_buffer: 0,
|
||||
truncate_output_to: None,
|
||||
expected_chunks: vec![decoded("!", &[b'!' as u32], &[visible(0)])],
|
||||
expected_full: decoded("!", &[b'!' as u32], &[visible(0)]),
|
||||
});
|
||||
}
|
||||
@@ -17,7 +17,7 @@ mod tiktoken;
|
||||
|
||||
pub use error::{Result, TokenizerError};
|
||||
pub use hf::HuggingFaceTokenizer;
|
||||
pub use incremental::IncrementalDecoder;
|
||||
pub use incremental::{DecodedText, IncrementalDecoder, TokenAnchor, TokenAttribution};
|
||||
pub use tekken::TekkenTokenizer;
|
||||
pub use tiktoken::TiktokenTokenizer;
|
||||
|
||||
|
||||
@@ -964,16 +964,16 @@ mod tests {
|
||||
for &id in &ids {
|
||||
decoder.push_token(id).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
|
||||
assert_eq!(output, text);
|
||||
assert_eq!(full_text, text);
|
||||
assert_eq!(full.text, text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -991,16 +991,16 @@ mod tests {
|
||||
for &id in &ids {
|
||||
decoder.push_token(id).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
let (last_chunk, full) = decoder.flush(None).unwrap();
|
||||
if let Some(chunk) = last_chunk {
|
||||
output.push_str(&chunk);
|
||||
output.push_str(&chunk.text);
|
||||
}
|
||||
|
||||
assert_eq!(output, text);
|
||||
assert_eq!(full_text, text);
|
||||
assert_eq!(full.text, text);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user