[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -167,6 +167,17 @@ class StreamingParserEngine:
|
||||
self.skip_tool_parsing = False
|
||||
self.reset(initial_state=initial_state)
|
||||
|
||||
@property
|
||||
def reasoning_token_count(self) -> int:
|
||||
return self._reasoning_token_count
|
||||
|
||||
def _record_reasoning_tokens(self, events: Sequence[SemanticEvent]) -> None:
|
||||
self._reasoning_token_count += sum(
|
||||
event.token_count
|
||||
for event in events
|
||||
if event.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
|
||||
def _reset_args_state(self) -> None:
|
||||
self._args_buffer: str = ""
|
||||
self._args_safe_end: int = 0
|
||||
@@ -186,6 +197,7 @@ class StreamingParserEngine:
|
||||
)
|
||||
self.tool_index = -1
|
||||
self._ever_had_token_ids = False
|
||||
self._reasoning_token_count = 0
|
||||
# DO NOT reset skip_tool_parsing here — callers set it before
|
||||
# calling methods that trigger reset() (e.g. extract_reasoning),
|
||||
# and clearing it silently breaks non-streaming tool-call-as-
|
||||
@@ -193,6 +205,7 @@ class StreamingParserEngine:
|
||||
self._scanner.reset()
|
||||
self._lexer.reset()
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
self._in_skipped_tool_span = False
|
||||
self._reset_args_state()
|
||||
|
||||
@@ -218,18 +231,30 @@ class StreamingParserEngine:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
return self._emit_for_state(delta_text)
|
||||
events = self._emit_for_state(
|
||||
delta_text, token_count=len(delta_token_ids)
|
||||
)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
|
||||
|
||||
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
|
||||
lex_tokens = self._lexer.feed(scanner_items[0].text)
|
||||
item = scanner_items[0]
|
||||
lex_tokens = self._lexer.feed(item.text, item.token_texts, item.token_count)
|
||||
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
|
||||
text = lex_tokens[0].value
|
||||
return self._emit_for_state(text)
|
||||
return self._process_lex_tokens(lex_tokens)
|
||||
events = self._emit_for_state(
|
||||
lex_tokens[0].value,
|
||||
token_count=lex_tokens[0].token_count,
|
||||
)
|
||||
else:
|
||||
events = self._process_lex_tokens(lex_tokens)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
return self._process_scanner_items(scanner_items)
|
||||
events = self._process_scanner_items(scanner_items)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
def _process_scanner_items(
|
||||
self, items: Sequence[LexerInput]
|
||||
@@ -240,7 +265,18 @@ class StreamingParserEngine:
|
||||
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
||||
events.extend(self._on_terminal(item.terminal, item.text))
|
||||
elif isinstance(item, TextChunk):
|
||||
events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
|
||||
if not item.text and item.token_count:
|
||||
events.extend(
|
||||
self._emit_for_state("", token_count=item.token_count)
|
||||
)
|
||||
else:
|
||||
events.extend(
|
||||
self._process_lex_tokens(
|
||||
self._lexer.feed(
|
||||
item.text, item.token_texts, item.token_count
|
||||
)
|
||||
)
|
||||
)
|
||||
return events
|
||||
|
||||
def finish(self) -> list[SemanticEvent]:
|
||||
@@ -285,11 +321,14 @@ class StreamingParserEngine:
|
||||
EventType.TEXT_CHUNK,
|
||||
value=self._message_header_buffer,
|
||||
tool_index=self.tool_index,
|
||||
token_count=self._message_header_token_count,
|
||||
)
|
||||
)
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
self.state = ParserState.CONTENT
|
||||
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
def parse_complete(self, text: str) -> list[SemanticEvent]:
|
||||
@@ -303,9 +342,11 @@ class StreamingParserEngine:
|
||||
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
|
||||
for tok in tokens:
|
||||
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
|
||||
events.extend(self._on_content(tok.value))
|
||||
events.extend(self._on_content(tok.value, tok.token_count))
|
||||
else:
|
||||
events.extend(self._on_terminal(tok.terminal, tok.value))
|
||||
events.extend(
|
||||
self._on_terminal(tok.terminal, tok.value, tok.token_count)
|
||||
)
|
||||
return events
|
||||
|
||||
_TOOL_STATES = frozenset(
|
||||
@@ -317,7 +358,9 @@ class StreamingParserEngine:
|
||||
}
|
||||
)
|
||||
|
||||
def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
|
||||
def _on_terminal(
|
||||
self, terminal: str, value: str, token_count: int = 0
|
||||
) -> list[SemanticEvent]:
|
||||
key = (self.state, terminal)
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
@@ -327,7 +370,7 @@ class StreamingParserEngine:
|
||||
# The projected skip state may not define the wrapper closer.
|
||||
if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
|
||||
self._in_skipped_tool_span = False
|
||||
return self._emit_for_state(value)
|
||||
return self._emit_for_state(value, token_count)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
# Inkling reuses one terminal for tool, text, and reasoning exits.
|
||||
@@ -345,6 +388,7 @@ class StreamingParserEngine:
|
||||
leaving_message_header = self.state == ParserState.MESSAGE_HEADER
|
||||
if leaving_message_header:
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
# A tool terminal that implicitly ends reasoning must report
|
||||
# that even from the header state, or the reasoning pass never
|
||||
# hands the block to the tool pass.
|
||||
@@ -381,13 +425,14 @@ class StreamingParserEngine:
|
||||
return []
|
||||
|
||||
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
|
||||
return self._emit_for_state(value)
|
||||
return self._emit_for_state(value, token_count)
|
||||
|
||||
return self._apply_transition(transition, value)
|
||||
return self._apply_transition(transition, value, token_count)
|
||||
|
||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
||||
def _emit_for_state(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self._message_header_buffer += text
|
||||
self._message_header_token_count += token_count
|
||||
return []
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_args_json:
|
||||
@@ -397,26 +442,36 @@ class StreamingParserEngine:
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=text,
|
||||
tool_index=self.tool_index,
|
||||
token_count=token_count,
|
||||
)
|
||||
]
|
||||
content_type = self.config.content_events.get(self.state)
|
||||
if content_type is not None:
|
||||
return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
|
||||
return [
|
||||
SemanticEvent(
|
||||
content_type,
|
||||
value=text,
|
||||
tool_index=self.tool_index,
|
||||
token_count=token_count,
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
def _on_content(self, text: str) -> list[SemanticEvent]:
|
||||
def _on_content(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||
if not text:
|
||||
return []
|
||||
return self._emit_for_state(text)
|
||||
return self._emit_for_state(text, token_count)
|
||||
|
||||
def _apply_transition(
|
||||
self,
|
||||
transition: Transition,
|
||||
value: str,
|
||||
token_count: int = 0,
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
previous_state = self.state
|
||||
message_header = ""
|
||||
message_header_token_count = 0
|
||||
|
||||
if (
|
||||
self.state == ParserState.TOOL_ARGS
|
||||
@@ -434,7 +489,9 @@ class StreamingParserEngine:
|
||||
|
||||
if previous_state == ParserState.MESSAGE_HEADER:
|
||||
message_header = self._message_header_buffer
|
||||
message_header_token_count = self._message_header_token_count
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
|
||||
self.state = transition.next_state
|
||||
|
||||
@@ -454,6 +511,12 @@ class StreamingParserEngine:
|
||||
event_type,
|
||||
value=event_value,
|
||||
tool_index=self.tool_index,
|
||||
token_count=(
|
||||
message_header_token_count
|
||||
if previous_state == ParserState.MESSAGE_HEADER
|
||||
and event_type == EventType.TEXT_CHUNK
|
||||
else token_count
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user