[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -367,6 +367,10 @@ class Parser:
|
||||
tool call extraction via internal stream state.
|
||||
"""
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Return the number of reasoning tokens in generated token IDs."""
|
||||
return 0
|
||||
|
||||
|
||||
class DelegatingParser(Parser):
|
||||
"""
|
||||
@@ -939,6 +943,12 @@ class DelegatingParser(Parser):
|
||||
|
||||
return delta_message
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Count reasoning tokens through the configured reasoning parser."""
|
||||
if self._reasoning_parser is None:
|
||||
return 0
|
||||
return self._reasoning_parser.count_reasoning_tokens(token_ids)
|
||||
|
||||
def _flush_engine_parsers(
|
||||
self, delta_message: DeltaMessage | None
|
||||
) -> DeltaMessage | None:
|
||||
|
||||
@@ -47,6 +47,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
|
||||
super().__init__(tokenizer, *args, **kwargs)
|
||||
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
|
||||
self._parser_engine_kwargs = kwargs
|
||||
self._counting_parser_engine: ParserEngine | None = None
|
||||
# TODO: Remove once Responses finalization reuses accumulated streaming
|
||||
# parser results instead of reparsing the complete output.
|
||||
self._streaming_count_valid = False
|
||||
|
||||
@contextmanager
|
||||
def _skip_tool_parsing(self) -> Iterator[None]:
|
||||
@@ -71,6 +76,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
self._streaming_count_valid = False
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning(model_output, request)
|
||||
|
||||
@@ -83,6 +89,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
self._streaming_count_valid = True
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
@@ -122,7 +129,18 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
return self._parser_engine.get_streaming_fallback_content(text, request)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||
if self._streaming_count_valid:
|
||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||
if not token_ids:
|
||||
return 0
|
||||
if self._counting_parser_engine is None:
|
||||
self._counting_parser_engine = self._parser_engine_cls(
|
||||
self.model_tokenizer, **self._parser_engine_kwargs
|
||||
) # type: ignore[call-arg]
|
||||
self._counting_parser_engine._single_pass_parse(
|
||||
self.model_tokenizer.decode(token_ids), token_ids
|
||||
)
|
||||
return self._counting_parser_engine.count_reasoning_tokens(token_ids)
|
||||
|
||||
|
||||
class ParserEngineToolAdapter(ToolParser):
|
||||
|
||||
@@ -24,3 +24,4 @@ class SemanticEvent:
|
||||
type: EventType
|
||||
value: str = ""
|
||||
tool_index: int = -1
|
||||
token_count: int = 0
|
||||
|
||||
@@ -24,6 +24,7 @@ class TerminalDef:
|
||||
class LexToken:
|
||||
terminal: str
|
||||
value: str
|
||||
token_count: int = 0
|
||||
|
||||
|
||||
class LexerShape:
|
||||
@@ -104,6 +105,7 @@ class IncrementalLexer:
|
||||
self.terminals = shape.terminals
|
||||
self.content_terminal = content_terminal
|
||||
self.buffer = ""
|
||||
self._token_counts: list[int] = []
|
||||
|
||||
self._literal_strings = shape.literal_strings
|
||||
self._max_literal_len = shape.max_literal_len
|
||||
@@ -114,15 +116,23 @@ class IncrementalLexer:
|
||||
|
||||
def reset(self) -> None:
|
||||
self.buffer = ""
|
||||
self._token_counts.clear()
|
||||
|
||||
def feed(self, text: str) -> list[LexToken]:
|
||||
def feed(
|
||||
self,
|
||||
text: str,
|
||||
token_texts: tuple[str, ...] = (),
|
||||
token_count: int = 0,
|
||||
) -> list[LexToken]:
|
||||
char_token_counts = self._char_token_counts(text, token_texts, token_count)
|
||||
if not self.buffer and self._has_only_literals and self._literal_first_chars:
|
||||
for ch in text:
|
||||
if ch in self._literal_first_chars:
|
||||
break
|
||||
else:
|
||||
return [LexToken(self.content_terminal, text)]
|
||||
return [LexToken(self.content_terminal, text, sum(char_token_counts))]
|
||||
self.buffer += text
|
||||
self._token_counts.extend(char_token_counts)
|
||||
return self._drain()
|
||||
|
||||
def flush(self) -> list[LexToken]:
|
||||
@@ -130,10 +140,46 @@ class IncrementalLexer:
|
||||
if self.buffer:
|
||||
tokens.extend(self._drain(final=True))
|
||||
if self.buffer:
|
||||
tokens.append(LexToken(self.content_terminal, self.buffer))
|
||||
tokens.append(
|
||||
LexToken(self.content_terminal, self.buffer, sum(self._token_counts))
|
||||
)
|
||||
self.buffer = ""
|
||||
self._token_counts.clear()
|
||||
return tokens
|
||||
|
||||
@staticmethod
|
||||
def _char_token_counts(
|
||||
text: str,
|
||||
token_texts: tuple[str, ...],
|
||||
token_count: int,
|
||||
) -> list[int]:
|
||||
counts = [0] * len(text)
|
||||
if not text:
|
||||
return counts
|
||||
if token_texts:
|
||||
pos = 0
|
||||
assigned = 0
|
||||
for token_text in token_texts:
|
||||
if not token_text:
|
||||
continue
|
||||
found = text.find(token_text, pos)
|
||||
if found < 0:
|
||||
continue
|
||||
counts[found] += 1
|
||||
assigned += 1
|
||||
pos = found + len(token_text)
|
||||
missing = token_count - assigned
|
||||
if missing > 0:
|
||||
counts[0] += missing
|
||||
elif token_count:
|
||||
counts[0] = token_count
|
||||
return counts
|
||||
|
||||
def _pop_token_count(self, length: int) -> int:
|
||||
token_count = sum(self._token_counts[:length])
|
||||
del self._token_counts[:length]
|
||||
return token_count
|
||||
|
||||
def _drain(self, *, final: bool = False) -> list[LexToken]:
|
||||
tokens: list[LexToken] = []
|
||||
first_chars = self._literal_first_chars
|
||||
@@ -150,8 +196,11 @@ class IncrementalLexer:
|
||||
has_potential = True
|
||||
break
|
||||
if not has_potential:
|
||||
tokens.append(LexToken(content_terminal, self.buffer))
|
||||
tokens.append(
|
||||
LexToken(content_terminal, self.buffer, sum(self._token_counts))
|
||||
)
|
||||
self.buffer = ""
|
||||
self._token_counts.clear()
|
||||
break
|
||||
|
||||
best_match: tuple[str, str, int] | None = None
|
||||
@@ -175,7 +224,13 @@ class IncrementalLexer:
|
||||
longer_match = True
|
||||
break
|
||||
if not longer_match:
|
||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
||||
tokens.append(
|
||||
LexToken(
|
||||
best_match[0],
|
||||
best_match[1],
|
||||
self._pop_token_count(best_match[2]),
|
||||
)
|
||||
)
|
||||
self.buffer = self.buffer[best_match[2] :]
|
||||
continue
|
||||
break
|
||||
@@ -183,15 +238,33 @@ class IncrementalLexer:
|
||||
break
|
||||
|
||||
if best_match is not None:
|
||||
tokens.append(LexToken(best_match[0], best_match[1]))
|
||||
tokens.append(
|
||||
LexToken(
|
||||
best_match[0],
|
||||
best_match[1],
|
||||
self._pop_token_count(best_match[2]),
|
||||
)
|
||||
)
|
||||
self.buffer = self.buffer[best_match[2] :]
|
||||
else:
|
||||
content_end = self._find_content_boundary()
|
||||
if content_end > 0:
|
||||
tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
|
||||
tokens.append(
|
||||
LexToken(
|
||||
content_terminal,
|
||||
self.buffer[:content_end],
|
||||
self._pop_token_count(content_end),
|
||||
)
|
||||
)
|
||||
self.buffer = self.buffer[content_end:]
|
||||
else:
|
||||
tokens.append(LexToken(content_terminal, self.buffer[0]))
|
||||
tokens.append(
|
||||
LexToken(
|
||||
content_terminal,
|
||||
self.buffer[0],
|
||||
self._pop_token_count(1),
|
||||
)
|
||||
)
|
||||
self.buffer = self.buffer[1:]
|
||||
|
||||
return tokens
|
||||
|
||||
@@ -625,23 +625,8 @@ class ParserEngine(Parser):
|
||||
return None
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
start_id = self._reasoning_start_token_id
|
||||
end_id = self._reasoning_end_token_id
|
||||
if start_id is None or end_id is None:
|
||||
return 0
|
||||
count = 0
|
||||
depth = 0
|
||||
for token_id in token_ids:
|
||||
if token_id == start_id:
|
||||
depth += 1
|
||||
continue
|
||||
if token_id == end_id:
|
||||
if depth > 0:
|
||||
depth -= 1
|
||||
continue
|
||||
if depth > 0:
|
||||
count += 1
|
||||
return count
|
||||
"""Return reasoning tokens observed by the parser engine so far."""
|
||||
return self._engine.reasoning_token_count
|
||||
|
||||
# ── Single-pass parse helper ────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -167,6 +167,17 @@ class StreamingParserEngine:
|
||||
self.skip_tool_parsing = False
|
||||
self.reset(initial_state=initial_state)
|
||||
|
||||
@property
|
||||
def reasoning_token_count(self) -> int:
|
||||
return self._reasoning_token_count
|
||||
|
||||
def _record_reasoning_tokens(self, events: Sequence[SemanticEvent]) -> None:
|
||||
self._reasoning_token_count += sum(
|
||||
event.token_count
|
||||
for event in events
|
||||
if event.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
|
||||
def _reset_args_state(self) -> None:
|
||||
self._args_buffer: str = ""
|
||||
self._args_safe_end: int = 0
|
||||
@@ -186,6 +197,7 @@ class StreamingParserEngine:
|
||||
)
|
||||
self.tool_index = -1
|
||||
self._ever_had_token_ids = False
|
||||
self._reasoning_token_count = 0
|
||||
# DO NOT reset skip_tool_parsing here — callers set it before
|
||||
# calling methods that trigger reset() (e.g. extract_reasoning),
|
||||
# and clearing it silently breaks non-streaming tool-call-as-
|
||||
@@ -193,6 +205,7 @@ class StreamingParserEngine:
|
||||
self._scanner.reset()
|
||||
self._lexer.reset()
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
self._in_skipped_tool_span = False
|
||||
self._reset_args_state()
|
||||
|
||||
@@ -218,18 +231,30 @@ class StreamingParserEngine:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
return self._emit_for_state(delta_text)
|
||||
events = self._emit_for_state(
|
||||
delta_text, token_count=len(delta_token_ids)
|
||||
)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
scanner_items = self._scanner.scan(delta_text, delta_token_ids)
|
||||
|
||||
if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
|
||||
lex_tokens = self._lexer.feed(scanner_items[0].text)
|
||||
item = scanner_items[0]
|
||||
lex_tokens = self._lexer.feed(item.text, item.token_texts, item.token_count)
|
||||
if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
|
||||
text = lex_tokens[0].value
|
||||
return self._emit_for_state(text)
|
||||
return self._process_lex_tokens(lex_tokens)
|
||||
events = self._emit_for_state(
|
||||
lex_tokens[0].value,
|
||||
token_count=lex_tokens[0].token_count,
|
||||
)
|
||||
else:
|
||||
events = self._process_lex_tokens(lex_tokens)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
return self._process_scanner_items(scanner_items)
|
||||
events = self._process_scanner_items(scanner_items)
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
def _process_scanner_items(
|
||||
self, items: Sequence[LexerInput]
|
||||
@@ -240,7 +265,18 @@ class StreamingParserEngine:
|
||||
events.extend(self._process_lex_tokens(self._lexer.flush()))
|
||||
events.extend(self._on_terminal(item.terminal, item.text))
|
||||
elif isinstance(item, TextChunk):
|
||||
events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
|
||||
if not item.text and item.token_count:
|
||||
events.extend(
|
||||
self._emit_for_state("", token_count=item.token_count)
|
||||
)
|
||||
else:
|
||||
events.extend(
|
||||
self._process_lex_tokens(
|
||||
self._lexer.feed(
|
||||
item.text, item.token_texts, item.token_count
|
||||
)
|
||||
)
|
||||
)
|
||||
return events
|
||||
|
||||
def finish(self) -> list[SemanticEvent]:
|
||||
@@ -285,11 +321,14 @@ class StreamingParserEngine:
|
||||
EventType.TEXT_CHUNK,
|
||||
value=self._message_header_buffer,
|
||||
tool_index=self.tool_index,
|
||||
token_count=self._message_header_token_count,
|
||||
)
|
||||
)
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
self.state = ParserState.CONTENT
|
||||
|
||||
self._record_reasoning_tokens(events)
|
||||
return events
|
||||
|
||||
def parse_complete(self, text: str) -> list[SemanticEvent]:
|
||||
@@ -303,9 +342,11 @@ class StreamingParserEngine:
|
||||
strict = self._token_id_terminal_names if self._ever_had_token_ids else None
|
||||
for tok in tokens:
|
||||
if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
|
||||
events.extend(self._on_content(tok.value))
|
||||
events.extend(self._on_content(tok.value, tok.token_count))
|
||||
else:
|
||||
events.extend(self._on_terminal(tok.terminal, tok.value))
|
||||
events.extend(
|
||||
self._on_terminal(tok.terminal, tok.value, tok.token_count)
|
||||
)
|
||||
return events
|
||||
|
||||
_TOOL_STATES = frozenset(
|
||||
@@ -317,7 +358,9 @@ class StreamingParserEngine:
|
||||
}
|
||||
)
|
||||
|
||||
def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
|
||||
def _on_terminal(
|
||||
self, terminal: str, value: str, token_count: int = 0
|
||||
) -> list[SemanticEvent]:
|
||||
key = (self.state, terminal)
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
@@ -327,7 +370,7 @@ class StreamingParserEngine:
|
||||
# The projected skip state may not define the wrapper closer.
|
||||
if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
|
||||
self._in_skipped_tool_span = False
|
||||
return self._emit_for_state(value)
|
||||
return self._emit_for_state(value, token_count)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
# Inkling reuses one terminal for tool, text, and reasoning exits.
|
||||
@@ -345,6 +388,7 @@ class StreamingParserEngine:
|
||||
leaving_message_header = self.state == ParserState.MESSAGE_HEADER
|
||||
if leaving_message_header:
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
# A tool terminal that implicitly ends reasoning must report
|
||||
# that even from the header state, or the reasoning pass never
|
||||
# hands the block to the tool pass.
|
||||
@@ -381,13 +425,14 @@ class StreamingParserEngine:
|
||||
return []
|
||||
|
||||
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
|
||||
return self._emit_for_state(value)
|
||||
return self._emit_for_state(value, token_count)
|
||||
|
||||
return self._apply_transition(transition, value)
|
||||
return self._apply_transition(transition, value, token_count)
|
||||
|
||||
def _emit_for_state(self, text: str) -> list[SemanticEvent]:
|
||||
def _emit_for_state(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||
if self.state == ParserState.MESSAGE_HEADER:
|
||||
self._message_header_buffer += text
|
||||
self._message_header_token_count += token_count
|
||||
return []
|
||||
if self.state == ParserState.TOOL_ARGS:
|
||||
if self.config.tool_args_json:
|
||||
@@ -397,26 +442,36 @@ class StreamingParserEngine:
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
value=text,
|
||||
tool_index=self.tool_index,
|
||||
token_count=token_count,
|
||||
)
|
||||
]
|
||||
content_type = self.config.content_events.get(self.state)
|
||||
if content_type is not None:
|
||||
return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
|
||||
return [
|
||||
SemanticEvent(
|
||||
content_type,
|
||||
value=text,
|
||||
tool_index=self.tool_index,
|
||||
token_count=token_count,
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
def _on_content(self, text: str) -> list[SemanticEvent]:
|
||||
def _on_content(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
|
||||
if not text:
|
||||
return []
|
||||
return self._emit_for_state(text)
|
||||
return self._emit_for_state(text, token_count)
|
||||
|
||||
def _apply_transition(
|
||||
self,
|
||||
transition: Transition,
|
||||
value: str,
|
||||
token_count: int = 0,
|
||||
) -> list[SemanticEvent]:
|
||||
events: list[SemanticEvent] = []
|
||||
previous_state = self.state
|
||||
message_header = ""
|
||||
message_header_token_count = 0
|
||||
|
||||
if (
|
||||
self.state == ParserState.TOOL_ARGS
|
||||
@@ -434,7 +489,9 @@ class StreamingParserEngine:
|
||||
|
||||
if previous_state == ParserState.MESSAGE_HEADER:
|
||||
message_header = self._message_header_buffer
|
||||
message_header_token_count = self._message_header_token_count
|
||||
self._message_header_buffer = ""
|
||||
self._message_header_token_count = 0
|
||||
|
||||
self.state = transition.next_state
|
||||
|
||||
@@ -454,6 +511,12 @@ class StreamingParserEngine:
|
||||
event_type,
|
||||
value=event_value,
|
||||
tool_index=self.tool_index,
|
||||
token_count=(
|
||||
message_header_token_count
|
||||
if previous_state == ParserState.MESSAGE_HEADER
|
||||
and event_type == EventType.TEXT_CHUNK
|
||||
else token_count
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ DROP_TERMINAL = "__DROP__"
|
||||
@dataclass(slots=True)
|
||||
class TextChunk:
|
||||
text: str
|
||||
token_texts: tuple[str, ...] = ()
|
||||
token_count: int = 0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -49,18 +51,30 @@ class TokenIDScanner:
|
||||
self.tokenizer = tokenizer
|
||||
self._token_text_cache: dict[int, str] = {}
|
||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||
self._deferred_prefix_token_counts: list[int] = []
|
||||
self._deferred_trailing_token_count = 0
|
||||
self._deferred_post_text: str = ""
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear mutable state for reuse. Preserves the token text cache."""
|
||||
self._deferred_terminals.clear()
|
||||
self._deferred_prefix_token_counts.clear()
|
||||
self._deferred_trailing_token_count = 0
|
||||
self._deferred_post_text = ""
|
||||
|
||||
def _decode_token(self, token_id: int) -> str:
|
||||
if token_id not in self._token_text_cache:
|
||||
self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
|
||||
token_text = self.tokenizer.decode([token_id])
|
||||
self._token_text_cache[token_id] = (
|
||||
token_text if isinstance(token_text, str) else ""
|
||||
)
|
||||
return self._token_text_cache[token_id]
|
||||
|
||||
def _decode_tokens(self, token_ids: Sequence[int]) -> tuple[str, ...]:
|
||||
if self.tokenizer is None:
|
||||
return ()
|
||||
return tuple(self._decode_token(tid) for tid in token_ids)
|
||||
|
||||
_EMPTY: tuple[LexerInput, ...] = ()
|
||||
|
||||
def scan(
|
||||
@@ -70,13 +84,23 @@ class TokenIDScanner:
|
||||
) -> Sequence[LexerInput]:
|
||||
prefix_items: list[LexerInput] = []
|
||||
effective_text = delta_text
|
||||
deferred_trailing_count = 0
|
||||
|
||||
if self._deferred_terminals:
|
||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
||||
prefix_items, effective_text, deferred_trailing_count = (
|
||||
self._resolve_deferred(delta_text, len(delta_token_ids))
|
||||
)
|
||||
|
||||
if not self.token_id_to_terminal:
|
||||
if effective_text:
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
token_texts = self._decode_tokens(delta_token_ids)
|
||||
prefix_items.append(
|
||||
TextChunk(
|
||||
effective_text,
|
||||
token_texts,
|
||||
deferred_trailing_count + len(token_texts),
|
||||
)
|
||||
)
|
||||
return prefix_items
|
||||
|
||||
has_special = False
|
||||
@@ -89,14 +113,23 @@ class TokenIDScanner:
|
||||
if not has_special:
|
||||
if effective_text:
|
||||
if not prefix_items:
|
||||
return [TextChunk(effective_text)]
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
token_texts = self._decode_tokens(delta_token_ids)
|
||||
return [TextChunk(effective_text, token_texts, len(token_texts))]
|
||||
token_texts = self._decode_tokens(delta_token_ids)
|
||||
prefix_items.append(
|
||||
TextChunk(
|
||||
effective_text,
|
||||
token_texts,
|
||||
deferred_trailing_count + len(token_texts),
|
||||
)
|
||||
)
|
||||
return prefix_items or self._EMPTY
|
||||
|
||||
token_texts = [self._decode_token(tid) for tid in delta_token_ids]
|
||||
decoded_token_texts = [self._decode_token(tid) for tid in delta_token_ids]
|
||||
|
||||
results: list[LexerInput] = []
|
||||
text_accum: list[str] = []
|
||||
token_text_accum: list[str] = []
|
||||
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
terminal = self.token_id_to_terminal.get(tid)
|
||||
@@ -104,16 +137,31 @@ class TokenIDScanner:
|
||||
if text_accum:
|
||||
joined = "".join(text_accum)
|
||||
if joined:
|
||||
results.append(TextChunk(joined))
|
||||
results.append(
|
||||
TextChunk(
|
||||
joined,
|
||||
tuple(token_text_accum),
|
||||
len(token_text_accum),
|
||||
)
|
||||
)
|
||||
text_accum.clear()
|
||||
results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
|
||||
token_text_accum.clear()
|
||||
results.append(
|
||||
PreLexedTerminal(terminal, tid, decoded_token_texts[idx])
|
||||
)
|
||||
else:
|
||||
text_accum.append(token_texts[idx])
|
||||
text_accum.append(decoded_token_texts[idx])
|
||||
token_text_accum.append(decoded_token_texts[idx])
|
||||
|
||||
if text_accum:
|
||||
joined = "".join(text_accum)
|
||||
if joined:
|
||||
results.append(TextChunk(joined))
|
||||
results.append(
|
||||
TextChunk(joined, tuple(token_text_accum), len(token_text_accum))
|
||||
)
|
||||
|
||||
if deferred_trailing_count:
|
||||
results.insert(0, TextChunk("", token_count=deferred_trailing_count))
|
||||
|
||||
if effective_text:
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
@@ -124,9 +172,15 @@ class TokenIDScanner:
|
||||
# transition before the preceding text has arrived. The
|
||||
# deferred terminals will be resolved against the actual
|
||||
# delta_text in a subsequent scan() or flushed by finish().
|
||||
prefix_token_count = 0
|
||||
for r in results:
|
||||
if isinstance(r, TextChunk):
|
||||
prefix_token_count += r.token_count
|
||||
if isinstance(r, PreLexedTerminal):
|
||||
self._deferred_terminals.append(r)
|
||||
self._deferred_prefix_token_counts.append(prefix_token_count)
|
||||
prefix_token_count = 0
|
||||
self._deferred_trailing_token_count += prefix_token_count
|
||||
results = []
|
||||
|
||||
return prefix_items + results
|
||||
@@ -136,16 +190,30 @@ class TokenIDScanner:
|
||||
return []
|
||||
results: list[LexerInput] = []
|
||||
if self._deferred_post_text:
|
||||
results.append(TextChunk(self._deferred_post_text))
|
||||
prefix_count = (
|
||||
self._deferred_prefix_token_counts[0]
|
||||
if self._deferred_prefix_token_counts
|
||||
else 0
|
||||
)
|
||||
results.append(
|
||||
TextChunk(self._deferred_post_text, token_count=prefix_count)
|
||||
)
|
||||
self._deferred_post_text = ""
|
||||
results.extend(self._deferred_terminals)
|
||||
if self._deferred_trailing_token_count:
|
||||
results.append(
|
||||
TextChunk("", token_count=self._deferred_trailing_token_count)
|
||||
)
|
||||
self._deferred_terminals.clear()
|
||||
self._deferred_prefix_token_counts.clear()
|
||||
self._deferred_trailing_token_count = 0
|
||||
return results
|
||||
|
||||
def _resolve_deferred(
|
||||
self,
|
||||
delta_text: str,
|
||||
) -> tuple[list[LexerInput], str]:
|
||||
current_token_count: int = 0,
|
||||
) -> tuple[list[LexerInput], str, int]:
|
||||
"""Resolve deferred terminals against new delta_text.
|
||||
|
||||
When a previous ``scan()`` deferred a terminal (its text hadn't
|
||||
@@ -160,7 +228,11 @@ class TokenIDScanner:
|
||||
should be scanned with the current delta's token IDs.
|
||||
"""
|
||||
deferred = self._deferred_terminals
|
||||
prefix_token_counts = self._deferred_prefix_token_counts
|
||||
trailing_token_count = self._deferred_trailing_token_count
|
||||
self._deferred_terminals = []
|
||||
self._deferred_prefix_token_counts = []
|
||||
self._deferred_trailing_token_count = 0
|
||||
|
||||
results: list[LexerInput] = []
|
||||
remaining = delta_text
|
||||
@@ -171,10 +243,13 @@ class TokenIDScanner:
|
||||
|
||||
# Duplicate-text deferred terminals resolve left-to-right via
|
||||
# find(); correct when each terminal text appears once in sequence.
|
||||
for terminal in deferred:
|
||||
for idx, terminal in enumerate(deferred):
|
||||
prefix_token_count = prefix_token_counts[idx]
|
||||
pos = remaining.find(terminal.text)
|
||||
if pos > 0:
|
||||
results.append(TextChunk(remaining[:pos]))
|
||||
results.append(
|
||||
TextChunk(remaining[:pos], token_count=prefix_token_count)
|
||||
)
|
||||
results.append(terminal)
|
||||
remaining = remaining[pos + len(terminal.text) :]
|
||||
elif pos == 0:
|
||||
@@ -185,10 +260,16 @@ class TokenIDScanner:
|
||||
# only the terminal provides a reliable split point.
|
||||
if remaining:
|
||||
self._deferred_post_text += remaining
|
||||
prefix_token_count += current_token_count
|
||||
remaining = ""
|
||||
self._deferred_terminals.append(terminal)
|
||||
self._deferred_prefix_token_counts.append(prefix_token_count)
|
||||
|
||||
return results, remaining
|
||||
if self._deferred_terminals:
|
||||
self._deferred_trailing_token_count = trailing_token_count
|
||||
trailing_token_count = 0
|
||||
|
||||
return results, remaining, trailing_token_count
|
||||
|
||||
def _recover_holdback_text(
|
||||
self,
|
||||
@@ -257,6 +338,16 @@ class TokenIDScanner:
|
||||
if not anchors:
|
||||
return [TextChunk(delta_text)]
|
||||
|
||||
token_groups: list[list[str]] = [[] for _ in range(len(anchors) + 1)]
|
||||
count_groups = [0] * (len(anchors) + 1)
|
||||
group_idx = 0
|
||||
for item in results:
|
||||
if isinstance(item, PreLexedTerminal):
|
||||
group_idx += 1
|
||||
else:
|
||||
token_groups[group_idx].extend(item.token_texts)
|
||||
count_groups[group_idx] += item.token_count
|
||||
|
||||
# Resolve positions right-to-left: each anchor gets the
|
||||
# rightmost occurrence that is still before the next anchor.
|
||||
positions: list[int] = [-1] * len(anchors)
|
||||
@@ -274,7 +365,13 @@ class TokenIDScanner:
|
||||
pos = positions[i]
|
||||
if pos >= consumed:
|
||||
if pos > consumed:
|
||||
new_results.append(TextChunk(delta_text[consumed:pos]))
|
||||
new_results.append(
|
||||
TextChunk(
|
||||
delta_text[consumed:pos],
|
||||
tuple(token_groups[i]),
|
||||
count_groups[i],
|
||||
)
|
||||
)
|
||||
new_results.append(anchor)
|
||||
consumed = pos + len(anchor.text)
|
||||
else:
|
||||
@@ -291,6 +388,13 @@ class TokenIDScanner:
|
||||
self._deferred_post_text += delta_text[consumed:]
|
||||
consumed = len(delta_text)
|
||||
self._deferred_terminals.append(anchor)
|
||||
self._deferred_prefix_token_counts.append(count_groups[i])
|
||||
if consumed < len(delta_text):
|
||||
new_results.append(TextChunk(delta_text[consumed:]))
|
||||
new_results.append(
|
||||
TextChunk(
|
||||
delta_text[consumed:],
|
||||
tuple(token_groups[-1]),
|
||||
count_groups[-1],
|
||||
)
|
||||
)
|
||||
return new_results
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.tool_parsers import ToolParser
|
||||
|
||||
@@ -17,10 +18,30 @@ logger = init_logger(__name__)
|
||||
|
||||
class ParserManager:
|
||||
"""
|
||||
Provides a unified Parser by composing individual reasoning and tool
|
||||
parsers from their respective registries.
|
||||
Provides a unified Parser from the reasoning and tool parser registries.
|
||||
|
||||
Parser engine adapters backed by the same engine are collapsed back into
|
||||
that engine. Other parser pairs are composed through ``DelegatingParser``.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _get_parser_engine_cls(
|
||||
parser_cls: type[object] | None,
|
||||
) -> type[ParserEngine] | None:
|
||||
if parser_cls is None:
|
||||
return None
|
||||
parser_engine_cls = getattr(parser_cls, "_parser_engine_cls", None)
|
||||
if parser_engine_cls is None:
|
||||
return None
|
||||
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
|
||||
if not isinstance(parser_engine_cls, type) or not issubclass(
|
||||
parser_engine_cls, ParserEngine
|
||||
):
|
||||
return None
|
||||
return parser_engine_cls
|
||||
|
||||
@classmethod
|
||||
def get_tool_parser(
|
||||
cls,
|
||||
@@ -84,8 +105,8 @@ class ParserManager:
|
||||
"""
|
||||
Get a Parser that handles both reasoning and tool parsing.
|
||||
|
||||
Composes individual reasoning and tool parsers into a single
|
||||
DelegatingParser subclass.
|
||||
Reuses a shared parser engine when possible, otherwise composes the
|
||||
individual parsers into a ``DelegatingParser`` subclass.
|
||||
|
||||
Args:
|
||||
tool_parser_name: The name of the tool parser.
|
||||
@@ -116,6 +137,11 @@ class ParserManager:
|
||||
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||
return HarmonyParser
|
||||
|
||||
reasoning_engine_cls = cls._get_parser_engine_cls(reasoning_parser_cls)
|
||||
tool_engine_cls = cls._get_parser_engine_cls(tool_parser_cls)
|
||||
if reasoning_engine_cls is not None and reasoning_engine_cls is tool_engine_cls:
|
||||
return reasoning_engine_cls
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
|
||||
|
||||
Reference in New Issue
Block a user