[Rust Frontend] Use token-attributed text in reasoning and unified parsers (#54884)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-04 02:05:05 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8bf39632b8
commit 21a2211040
22 changed files with 1012 additions and 350 deletions
@@ -67,9 +67,63 @@ impl DecodedText {
}
}
/// Return true when both the text and the attributions are empty.
pub fn is_empty(&self) -> bool {
self.text.is_empty() && self.attributions.is_empty()
}
/// Take the decoded text and attributions, leaving `self` empty.
pub fn take(&mut self) -> Self {
take(self)
}
/// Clear the text and attributions, keeping the allocations.
pub fn clear(&mut self) {
self.text.clear();
self.attributions.clear();
}
/// Split off and return the prefix `[0, len)`; the suffix stays in `self`,
/// with its token anchors rebased by `-len`.
///
/// Anchors follow the first-byte ownership rule: a `Visible` token moves to
/// the prefix when its first byte lies inside the prefix, so a token
/// straddling the split belongs to the span holding its first byte. A
/// `ZeroWidth` token moves to the prefix when its offset is `<= len`,
/// matching the `take_ready` cutoff rule and letting
/// `drain_prefix(text.len())` take trailing zero-width tokens. Relative
/// order across anchor kinds is preserved.
///
/// Panics if `len` is not a char boundary, via `String::drain`.
#[must_use]
pub fn drain_prefix(&mut self, len: usize) -> DecodedText {
let text: String = self.text.drain(..len).collect();
let len = offset_as_u32(len);
let mut prefix = DecodedText {
text,
attributions: SmallVec::new(),
};
for attribution in take(&mut self.attributions) {
let in_prefix = match attribution.anchor {
TokenAnchor::Visible { byte_offset } => byte_offset < len,
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= len,
};
if in_prefix {
prefix.attributions.push(attribution);
} else {
self.attributions.push(TokenAttribution {
token_id: attribution.token_id,
anchor: attribution.anchor.offset_by(-i64::from(len)),
});
}
}
prefix
}
/// Append another decoded fragment, rebasing its token anchors.
pub fn append(&mut self, other: Self) {
if self.text.is_empty() && self.attributions.is_empty() {
if self.is_empty() {
*self = other;
return;
}
@@ -169,6 +169,145 @@ fn decoded_text_append_preserves_zero_width_only_receiver() {
);
}
#[test]
fn decoded_text_drain_prefix_moves_anchors_before_boundary() {
let mut dt = decoded("ab.cd", &[1, 2, 3], &[visible(0), visible(1), visible(3)]);
let prefix = dt.drain_prefix(2);
assert_eq!(prefix, decoded("ab", &[1, 2], &[visible(0), visible(1)]));
assert_eq!(dt, decoded(".cd", &[3], &[visible(1)]));
}
#[test]
fn decoded_text_drain_prefix_keeps_straddling_token_in_prefix() {
// One token decodes to "ab"; a split inside it keeps the token in the
// prefix, the span holding its first byte.
let mut dt = decoded("ab", &[1], &[visible(0)]);
let prefix = dt.drain_prefix(1);
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
assert_eq!(dt, decoded("b", &[], &[]));
}
#[test]
fn decoded_text_drain_prefix_visible_anchor_at_boundary_goes_to_suffix() {
let mut dt = decoded("ab", &[1, 2], &[visible(0), visible(1)]);
let prefix = dt.drain_prefix(1);
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
assert_eq!(dt, decoded("b", &[2], &[visible(0)]));
}
#[test]
fn decoded_text_drain_prefix_zero_width_at_boundary_goes_to_prefix() {
let mut dt = decoded("ab", &[1, 2], &[zero_width(1), visible(1)]);
let prefix = dt.drain_prefix(1);
assert_eq!(prefix, decoded("a", &[1], &[zero_width(1)]));
assert_eq!(dt, decoded("b", &[2], &[visible(0)]));
}
#[test]
fn decoded_text_drain_prefix_shared_anchor_group_moves_together() {
// A byte-fallback group jointly decoding one character shares one anchor;
// the whole group stays on whichever side holds that first byte.
let mut dt = decoded(
"a你",
&[1, 2, 3, 4],
&[visible(0), visible(1), visible(1), visible(1)],
);
let prefix = dt.drain_prefix(1);
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
assert_eq!(
dt,
decoded("你", &[2, 3, 4], &[visible(0), visible(0), visible(0)])
);
let prefix = dt.drain_prefix(3);
assert_eq!(
prefix,
decoded("你", &[2, 3, 4], &[visible(0), visible(0), visible(0)])
);
assert_eq!(dt, decoded("", &[], &[]));
}
#[test]
fn decoded_text_drain_prefix_zero_drains_leading_zero_width() {
let mut dt = decoded("ab", &[1, 2], &[zero_width(0), visible(0)]);
let prefix = dt.drain_prefix(0);
assert_eq!(prefix, decoded("", &[1], &[zero_width(0)]));
assert_eq!(dt, decoded("ab", &[2], &[visible(0)]));
}
#[test]
fn decoded_text_drain_prefix_full_length_takes_trailing_zero_width() {
let mut dt = decoded("ab", &[1, 2, 3], &[visible(0), visible(1), zero_width(2)]);
let prefix = dt.drain_prefix(2);
assert_eq!(
prefix,
decoded("ab", &[1, 2, 3], &[visible(0), visible(1), zero_width(2)])
);
assert_eq!(dt, decoded("", &[], &[]));
}
#[test]
fn decoded_text_drain_prefix_preserves_order_across_anchor_kinds() {
let mut dt = decoded(
"ab",
&[1, 2, 3, 4],
&[visible(0), zero_width(1), visible(1), zero_width(2)],
);
let prefix = dt.drain_prefix(1);
assert_eq!(prefix, decoded("a", &[1, 2], &[visible(0), zero_width(1)]));
assert_eq!(dt, decoded("b", &[3, 4], &[visible(0), zero_width(1)]));
}
#[test]
fn decoded_text_drain_prefix_then_append_reconstructs_original() {
// ASCII, a byte-fallback group, and mid and trailing zero-width tokens.
let original = decoded(
"a你b",
&[1, 2, 3, 4, 5, 6, 7],
&[
visible(0),
visible(1),
visible(1),
visible(1),
zero_width(4),
visible(4),
zero_width(5),
],
);
for len in [0, 1, 4, 5] {
let mut rest = original.clone();
let mut prefix = rest.drain_prefix(len);
prefix.append(rest);
assert_eq!(prefix, original, "split at byte {len}");
}
}
#[test]
#[should_panic(expected = "is_char_boundary")]
fn decoded_text_drain_prefix_panics_off_char_boundary() {
let mut dt = decoded("a你", &[1, 2], &[visible(0), visible(1)]);
let _ = dt.drain_prefix(2);
}
struct AttributionCase<'a> {
tokenizer: &'a dyn Tokenizer,
prompt_token_ids: Vec<u32>,