[Frontend] Preserve bare Inkling text in Python and Rust parsers (#50403)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-30 12:25:10 -07:00
committed by GitHub
parent 12a34a6bc7
commit 629a938a92
4 changed files with 146 additions and 20 deletions
+29 -1
View File
@@ -185,6 +185,7 @@ class StreamingParserEngine:
# implicit-reasoning-end (content returns None).
self._scanner.reset()
self._lexer.reset()
self._message_header_buffer = ""
self._reset_args_state()
def feed(
@@ -270,6 +271,15 @@ class StreamingParserEngine:
)
self.state = ParserState.CONTENT
elif self.state == ParserState.MESSAGE_HEADER:
if self._message_header_buffer:
events.append(
SemanticEvent(
EventType.TEXT_CHUNK,
value=self._message_header_buffer,
tool_index=self.tool_index,
)
)
self._message_header_buffer = ""
self.state = ParserState.CONTENT
return events
@@ -311,6 +321,7 @@ class StreamingParserEngine:
if self.skip_tool_parsing and terminal in self._tool_terminals:
if self.state == ParserState.MESSAGE_HEADER:
self.state = ParserState.CONTENT
self._message_header_buffer = ""
return [
SemanticEvent(
EventType.TEXT_CHUNK,
@@ -345,6 +356,9 @@ class StreamingParserEngine:
return self._apply_transition(transition, value)
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
if self.state == ParserState.MESSAGE_HEADER:
self._message_header_buffer += text
return []
if self.state == ParserState.TOOL_ARGS:
if self.config.tool_args_json:
return self._feed_args_text(text)
@@ -371,6 +385,8 @@ class StreamingParserEngine:
value: str,
) -> list[SemanticEvent]:
events: list[SemanticEvent] = []
previous_state = self.state
message_header = ""
if (
self.state == ParserState.TOOL_ARGS
@@ -386,15 +402,27 @@ class StreamingParserEngine:
)
self._args_buffer = ""
if previous_state == ParserState.MESSAGE_HEADER:
message_header = self._message_header_buffer
self._message_header_buffer = ""
self.state = transition.next_state
for event_type in transition.events:
if event_type == EventType.TOOL_CALL_START:
self.tool_index += 1
event_value = (
message_header
if previous_state == ParserState.MESSAGE_HEADER
and event_type == EventType.TEXT_CHUNK
else value
)
if event_type == EventType.TEXT_CHUNK and not event_value:
continue
events.append(
SemanticEvent(
event_type,
value=value,
value=event_value,
tool_index=self.tool_index,
)
)