[Bugfix][Parser] Fix U+FFFD leak at reasoning-to-content transition in engine parsers (#46159)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning
2026-06-19 22:09:28 +00:00
committed by GitHub
parent 4a083cc858
commit 859e4d436b
5 changed files with 191 additions and 7 deletions
+3 -4
View File
@@ -794,11 +794,10 @@ class DelegatingParser(Parser):
reasoning_transitioned = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if self._engine_based:
flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]
current_text = (
self.model_tokenizer.decode(current_token_ids)
if current_token_ids
else ""
)
(delta_message.content if delta_message else None) or ""
) + ((flush_delta.content if flush_delta else None) or "")
if delta_message and self._tool_parser is not None:
delta_message.content = None
else: