[Bugfix][Parser] Fix U+FFFD leak at reasoning-to-content transition in engine parsers (#46159)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning
2026-06-19 22:09:28 +00:00
committed by GitHub
parent 4a083cc858
commit 859e4d436b
5 changed files with 191 additions and 7 deletions
+3 -4
View File
@@ -794,11 +794,10 @@ class DelegatingParser(Parser):
reasoning_transitioned = True
current_token_ids = self.extract_content_ids(delta_token_ids)
if self._engine_based:
flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]
current_text = (
self.model_tokenizer.decode(current_token_ids)
if current_token_ids
else ""
)
(delta_message.content if delta_message else None) or ""
) + ((flush_delta.content if flush_delta else None) or "")
if delta_message and self._tool_parser is not None:
delta_message.content = None
else:
+3 -1
View File
@@ -172,7 +172,9 @@ class ParserEngine(Parser):
def finish_streaming(self) -> DeltaMessage | None:
events = self._engine.finish()
return self._events_to_delta(events) if events else None
if events or self._deferred_content:
return self._events_to_delta(events, finished=True)
return None
def _reset(self, initial_state: ParserState | None = None) -> None:
self._engine.reset(initial_state=initial_state)