[Parser] Fix: correct parser frontend handling of reasoning end and boundary tokens (#56635)

Signed-off-by: DEV TYAGI <devtyagi3909@gmail.com>
This commit is contained in:
dev
2026-09-16 01:17:22 -07:00
committed by GitHub
parent 9ca6dbba71
commit 8be5205abb
3 changed files with 135 additions and 6 deletions
+19
View File
@@ -672,11 +672,30 @@ class ParserEngine(Parser):
return not wait_for_reasoning
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
config = self.parser_engine_config
wait_for_reasoning = config.wait_for_reasoning
if wait_for_reasoning is None:
wait_for_reasoning = config.initial_state is ParserState.REASONING
if not wait_for_reasoning:
return input_ids
end_id = self._reasoning_end_token_id
if end_id is not None:
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] == end_id:
return input_ids[i + 1 :]
end_ids = self._reasoning_end_token_ids
if end_ids:
turn_start = 0
boundary_ids = self._turn_boundary_token_ids
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] in boundary_ids:
turn_start = i + 1
break
for i in range(turn_start, len(input_ids)):
if input_ids[i] in end_ids:
return input_ids[i:]
return input_ids
def get_streaming_fallback_content(