[Rust Frontend] Use token-attributed text in reasoning and unified parsers (#54884)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8bf39632b8
commit
21a2211040
@@ -67,9 +67,63 @@ impl DecodedText {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return true when both the text and the attributions are empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.text.is_empty() && self.attributions.is_empty()
|
||||
}
|
||||
|
||||
/// Take the decoded text and attributions, leaving `self` empty.
|
||||
pub fn take(&mut self) -> Self {
|
||||
take(self)
|
||||
}
|
||||
|
||||
/// Clear the text and attributions, keeping the allocations.
|
||||
pub fn clear(&mut self) {
|
||||
self.text.clear();
|
||||
self.attributions.clear();
|
||||
}
|
||||
|
||||
/// Split off and return the prefix `[0, len)`; the suffix stays in `self`,
|
||||
/// with its token anchors rebased by `-len`.
|
||||
///
|
||||
/// Anchors follow the first-byte ownership rule: a `Visible` token moves to
|
||||
/// the prefix when its first byte lies inside the prefix, so a token
|
||||
/// straddling the split belongs to the span holding its first byte. A
|
||||
/// `ZeroWidth` token moves to the prefix when its offset is `<= len`,
|
||||
/// matching the `take_ready` cutoff rule and letting
|
||||
/// `drain_prefix(text.len())` take trailing zero-width tokens. Relative
|
||||
/// order across anchor kinds is preserved.
|
||||
///
|
||||
/// Panics if `len` is not a char boundary, via `String::drain`.
|
||||
#[must_use]
|
||||
pub fn drain_prefix(&mut self, len: usize) -> DecodedText {
|
||||
let text: String = self.text.drain(..len).collect();
|
||||
let len = offset_as_u32(len);
|
||||
|
||||
let mut prefix = DecodedText {
|
||||
text,
|
||||
attributions: SmallVec::new(),
|
||||
};
|
||||
for attribution in take(&mut self.attributions) {
|
||||
let in_prefix = match attribution.anchor {
|
||||
TokenAnchor::Visible { byte_offset } => byte_offset < len,
|
||||
TokenAnchor::ZeroWidth { byte_offset } => byte_offset <= len,
|
||||
};
|
||||
if in_prefix {
|
||||
prefix.attributions.push(attribution);
|
||||
} else {
|
||||
self.attributions.push(TokenAttribution {
|
||||
token_id: attribution.token_id,
|
||||
anchor: attribution.anchor.offset_by(-i64::from(len)),
|
||||
});
|
||||
}
|
||||
}
|
||||
prefix
|
||||
}
|
||||
|
||||
/// Append another decoded fragment, rebasing its token anchors.
|
||||
pub fn append(&mut self, other: Self) {
|
||||
if self.text.is_empty() && self.attributions.is_empty() {
|
||||
if self.is_empty() {
|
||||
*self = other;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -169,6 +169,145 @@ fn decoded_text_append_preserves_zero_width_only_receiver() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_moves_anchors_before_boundary() {
|
||||
let mut dt = decoded("ab.cd", &[1, 2, 3], &[visible(0), visible(1), visible(3)]);
|
||||
|
||||
let prefix = dt.drain_prefix(2);
|
||||
|
||||
assert_eq!(prefix, decoded("ab", &[1, 2], &[visible(0), visible(1)]));
|
||||
assert_eq!(dt, decoded(".cd", &[3], &[visible(1)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_keeps_straddling_token_in_prefix() {
|
||||
// One token decodes to "ab"; a split inside it keeps the token in the
|
||||
// prefix, the span holding its first byte.
|
||||
let mut dt = decoded("ab", &[1], &[visible(0)]);
|
||||
|
||||
let prefix = dt.drain_prefix(1);
|
||||
|
||||
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
|
||||
assert_eq!(dt, decoded("b", &[], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_visible_anchor_at_boundary_goes_to_suffix() {
|
||||
let mut dt = decoded("ab", &[1, 2], &[visible(0), visible(1)]);
|
||||
|
||||
let prefix = dt.drain_prefix(1);
|
||||
|
||||
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
|
||||
assert_eq!(dt, decoded("b", &[2], &[visible(0)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_zero_width_at_boundary_goes_to_prefix() {
|
||||
let mut dt = decoded("ab", &[1, 2], &[zero_width(1), visible(1)]);
|
||||
|
||||
let prefix = dt.drain_prefix(1);
|
||||
|
||||
assert_eq!(prefix, decoded("a", &[1], &[zero_width(1)]));
|
||||
assert_eq!(dt, decoded("b", &[2], &[visible(0)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_shared_anchor_group_moves_together() {
|
||||
// A byte-fallback group jointly decoding one character shares one anchor;
|
||||
// the whole group stays on whichever side holds that first byte.
|
||||
let mut dt = decoded(
|
||||
"a你",
|
||||
&[1, 2, 3, 4],
|
||||
&[visible(0), visible(1), visible(1), visible(1)],
|
||||
);
|
||||
|
||||
let prefix = dt.drain_prefix(1);
|
||||
|
||||
assert_eq!(prefix, decoded("a", &[1], &[visible(0)]));
|
||||
assert_eq!(
|
||||
dt,
|
||||
decoded("你", &[2, 3, 4], &[visible(0), visible(0), visible(0)])
|
||||
);
|
||||
|
||||
let prefix = dt.drain_prefix(3);
|
||||
|
||||
assert_eq!(
|
||||
prefix,
|
||||
decoded("你", &[2, 3, 4], &[visible(0), visible(0), visible(0)])
|
||||
);
|
||||
assert_eq!(dt, decoded("", &[], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_zero_drains_leading_zero_width() {
|
||||
let mut dt = decoded("ab", &[1, 2], &[zero_width(0), visible(0)]);
|
||||
|
||||
let prefix = dt.drain_prefix(0);
|
||||
|
||||
assert_eq!(prefix, decoded("", &[1], &[zero_width(0)]));
|
||||
assert_eq!(dt, decoded("ab", &[2], &[visible(0)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_full_length_takes_trailing_zero_width() {
|
||||
let mut dt = decoded("ab", &[1, 2, 3], &[visible(0), visible(1), zero_width(2)]);
|
||||
|
||||
let prefix = dt.drain_prefix(2);
|
||||
|
||||
assert_eq!(
|
||||
prefix,
|
||||
decoded("ab", &[1, 2, 3], &[visible(0), visible(1), zero_width(2)])
|
||||
);
|
||||
assert_eq!(dt, decoded("", &[], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_preserves_order_across_anchor_kinds() {
|
||||
let mut dt = decoded(
|
||||
"ab",
|
||||
&[1, 2, 3, 4],
|
||||
&[visible(0), zero_width(1), visible(1), zero_width(2)],
|
||||
);
|
||||
|
||||
let prefix = dt.drain_prefix(1);
|
||||
|
||||
assert_eq!(prefix, decoded("a", &[1, 2], &[visible(0), zero_width(1)]));
|
||||
assert_eq!(dt, decoded("b", &[3, 4], &[visible(0), zero_width(1)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_text_drain_prefix_then_append_reconstructs_original() {
|
||||
// ASCII, a byte-fallback group, and mid and trailing zero-width tokens.
|
||||
let original = decoded(
|
||||
"a你b",
|
||||
&[1, 2, 3, 4, 5, 6, 7],
|
||||
&[
|
||||
visible(0),
|
||||
visible(1),
|
||||
visible(1),
|
||||
visible(1),
|
||||
zero_width(4),
|
||||
visible(4),
|
||||
zero_width(5),
|
||||
],
|
||||
);
|
||||
|
||||
for len in [0, 1, 4, 5] {
|
||||
let mut rest = original.clone();
|
||||
let mut prefix = rest.drain_prefix(len);
|
||||
prefix.append(rest);
|
||||
assert_eq!(prefix, original, "split at byte {len}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "is_char_boundary")]
|
||||
fn decoded_text_drain_prefix_panics_off_char_boundary() {
|
||||
let mut dt = decoded("a你", &[1, 2], &[visible(0), visible(1)]);
|
||||
|
||||
let _ = dt.drain_prefix(2);
|
||||
}
|
||||
|
||||
struct AttributionCase<'a> {
|
||||
tokenizer: &'a dyn Tokenizer,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
|
||||
Reference in New Issue
Block a user