[Frontend] Preserve bare Inkling text in Python and Rust parsers (#50403)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -185,6 +185,7 @@ class StreamingParserEngine:
|
||||
# implicit-reasoning-end (content returns None).
|
||||
self._scanner.reset()
|
||||
self._lexer.reset()
|
||||
self._message_header_buffer = ""
|
||||
self._reset_args_state()
|
||||
|
||||
def feed(
|
||||
@@ -270,6 +271,15 @@ class StreamingParserEngine:
|
||||
)
|
||||
self.state = ParserState.CONTENT
|
||||
elif self.state == ParserState.MESSAGE_HEADER:
|
||||
if self._message_header_buffer:
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
EventType.TEXT_CHUNK,
|
||||
value=self._message_header_buffer,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
self._message_header_buffer = ""
|
||||
self.state = ParserState.CONTENT
|
||||
|
||||
return events
|
||||
@@ -311,6 +321,7 @@ class StreamingParserEngine:
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self.state = ParserState.CONTENT
|
||||
self._message_header_buffer = ""
|
||||
return [
|
||||
SemanticEvent(
|
||||
EventType.TEXT_CHUNK,
|
||||
@@ -345,6 +356,9 @@ class StreamingParserEngine:
|
||||
return self._apply_transition(transition, value)
|
||||
|
||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self._message_header_buffer += text
|
||||
return []
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_args_json:
|
||||
return self._feed_args_text(text)
|
||||
@@ -371,6 +385,8 @@ class StreamingParserEngine:
|
||||
value: str,
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
previous_state = self.state
|
||||
message_header = ""
|
||||
|
||||
if (
|
||||
self.state == ParserState.TOOL_ARGS
|
||||
@@ -386,15 +402,27 @@ class StreamingParserEngine:
|
||||
)
|
||||
self._args_buffer = ""
|
||||
|
||||
if previous_state == ParserState.MESSAGE_HEADER:
|
||||
message_header = self._message_header_buffer
|
||||
self._message_header_buffer = ""
|
||||
|
||||
self.state = transition.next_state
|
||||
|
||||
for event_type in transition.events:
|
||||
if event_type == EventType.TOOL_CALL_START:
|
||||
self.tool_index += 1
|
||||
event_value = (
|
||||
message_header
|
||||
if previous_state == ParserState.MESSAGE_HEADER
|
||||
and event_type == EventType.TEXT_CHUNK
|
||||
else value
|
||||
)
|
||||
if event_type == EventType.TEXT_CHUNK and not event_value:
|
||||
continue
|
||||
events.append(
|
||||
SemanticEvent(
|
||||
event_type,
|
||||
value=value,
|
||||
value=event_value,
|
||||
tool_index=self.tool_index,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user