[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -47,6 +47,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
|
||||
super().__init__(tokenizer, *args, **kwargs)
|
||||
self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
|
||||
self._parser_engine_kwargs = kwargs
|
||||
self._counting_parser_engine: ParserEngine | None = None
|
||||
# TODO: Remove once Responses finalization reuses accumulated streaming
|
||||
# parser results instead of reparsing the complete output.
|
||||
self._streaming_count_valid = False
|
||||
|
||||
@contextmanager
|
||||
def _skip_tool_parsing(self) -> Iterator[None]:
|
||||
@@ -71,6 +76,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
self._streaming_count_valid = False
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning(model_output, request)
|
||||
|
||||
@@ -83,6 +89,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
self._streaming_count_valid = True
|
||||
with self._skip_tool_parsing():
|
||||
return self._parser_engine.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
@@ -122,7 +129,18 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
return self._parser_engine.get_streaming_fallback_content(text, request)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||
if self._streaming_count_valid:
|
||||
return self._parser_engine.count_reasoning_tokens(token_ids)
|
||||
if not token_ids:
|
||||
return 0
|
||||
if self._counting_parser_engine is None:
|
||||
self._counting_parser_engine = self._parser_engine_cls(
|
||||
self.model_tokenizer, **self._parser_engine_kwargs
|
||||
) # type: ignore[call-arg]
|
||||
self._counting_parser_engine._single_pass_parse(
|
||||
self.model_tokenizer.decode(token_ids), token_ids
|
||||
)
|
||||
return self._counting_parser_engine.count_reasoning_tokens(token_ids)
|
||||
|
||||
|
||||
class ParserEngineToolAdapter(ToolParser):
|
||||
|
||||
Reference in New Issue
Block a user