[Structured Output][Reasoning] Improves decoding throughput for models using single-token reasoning endings. (#30056)
This commit is contained in:
@@ -63,6 +63,31 @@ class ReasoningParser:
|
||||
True if the reasoning content ends in the input_ids.
|
||||
"""
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the reasoning content ends in the input_ids on a
|
||||
decode step.
|
||||
|
||||
It is used in structured engines like `xgrammar` to check if the
|
||||
reasoning content ends in the model output during a decode step.
|
||||
`input_ids` the entire model output and `delta_ids` are the last few
|
||||
computed tokens of the model output (like during a decode step).
|
||||
|
||||
Parameters:
|
||||
input_ids: list[int]
|
||||
The entire model output.
|
||||
delta_ids: list[int]
|
||||
The last few computed tokens of the model output at the current decode step.
|
||||
|
||||
Returns:
|
||||
bool
|
||||
True if the reasoning content ends in the `delta_ids` on a
|
||||
decode step.
|
||||
"""
|
||||
return self.is_reasoning_end(input_ids)
|
||||
|
||||
@abstractmethod
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
"""
|
||||
|
||||
@@ -74,6 +74,12 @@ class BaseThinkingReasoningParser(ReasoningParser):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
end_token_id = self.end_token_id
|
||||
return end_token_id in delta_ids
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
"""
|
||||
Extract the content after the end tokens
|
||||
|
||||
@@ -35,6 +35,11 @@ class DeepSeekV3ReasoningParser(ReasoningParser):
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
return self._parser.is_reasoning_end(input_ids)
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
return self._parser.is_reasoning_end_streaming(input_ids, delta_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
return self._parser.extract_content_ids(input_ids)
|
||||
|
||||
|
||||
@@ -56,6 +56,11 @@ class Holo2ReasoningParser(ReasoningParser):
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
return self._parser.is_reasoning_end(input_ids)
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
return self._parser.is_reasoning_end_streaming(input_ids, delta_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
return self._parser.extract_content_ids(input_ids)
|
||||
|
||||
|
||||
@@ -32,6 +32,11 @@ class IdentityReasoningParser(ReasoningParser):
|
||||
# Always return True, since we never treat reasoning specially
|
||||
return True
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
# Identity: return all tokens as content
|
||||
return input_ids
|
||||
|
||||
@@ -339,7 +339,9 @@ class StructuredOutputManager:
|
||||
return True
|
||||
|
||||
# Check if reasoning ends in *this* step
|
||||
if self.reasoner.is_reasoning_end(request.all_token_ids):
|
||||
if self.reasoner.is_reasoning_end_streaming(
|
||||
request.all_token_ids, request.all_token_ids[request.num_computed_tokens :]
|
||||
):
|
||||
# Reasoning just ended, so we shouldn't advance til
|
||||
# next pass
|
||||
structured_req.reasoning_ended = True
|
||||
|
||||
Reference in New Issue
Block a user