[Bugfix][Parser] Fix U+FFFD leak at reasoning-to-content transition in engine parsers (#46159)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning
2026-06-19 22:09:28 +00:00
committed by GitHub
parent 4a083cc858
commit 859e4d436b
5 changed files with 191 additions and 7 deletions
+3
View File
@@ -96,6 +96,9 @@ class MockTokenizer:
return "".join(parts)
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
"""Build a mock tokenizer from a sample's vocab and token data."""
return MockTokenizer(