[Perf] Eliminate full-history reasoning scans for structured outputs (#55223)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Flora Feng
2026-09-08 14:07:16 -07:00
committed by GitHub
co-authored by Wentao Ye
parent 73f61d12aa
commit 29f46ac475
8 changed files with 475 additions and 42 deletions
+39
View File
@@ -152,6 +152,9 @@ class ParserEngine(Parser):
for token in parser_engine_config.turn_boundary_tokens
if (token_id := vocab.get(token)) is not None
)
self._reasoning_end_token_ids: frozenset[int] = (
self._derive_reasoning_end_token_ids(parser_engine_config, vocab)
)
@property
def reasoning_start_str(self) -> str | None:
@@ -608,6 +611,42 @@ class ParserEngine(Parser):
# ── Reasoning state queries ───────────────────────────────────────
@staticmethod
def _derive_reasoning_end_token_ids(
config: ParserEngineConfig, vocab: dict[str, int]
) -> frozenset[int]:
end_terminals: dict[str, ParserState] = {}
for (state, terminal), transition in config.transitions.items():
if state != ParserState.REASONING:
continue
if EventType.REASONING_END in transition.events:
end_terminals.setdefault(terminal, transition.next_state)
elif transition.next_state != ParserState.REASONING:
return frozenset()
token_ids: set[int] = set()
for terminal, next_state in end_terminals.items():
text = config.token_id_terminals.get(terminal)
token_id = vocab.get(text) if text is not None else None
if token_id is not None:
token_ids.add(token_id)
elif next_state == ParserState.CONTENT:
return frozenset()
return frozenset(token_ids)
@property
def reasoning_end_token_ids(self) -> frozenset[int]:
return self._reasoning_end_token_ids
def find_reasoning_end_offset(self, token_ids: Sequence[int]) -> int | None:
end_ids = self._reasoning_end_token_ids
if not end_ids:
return None
for offset, token_id in enumerate(token_ids):
if token_id in end_ids:
return offset
return None
def is_reasoning_end(self, input_ids: list[int]) -> bool:
end_id = self._reasoning_end_token_id
start_id = self._reasoning_start_token_id