[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-08-15 06:51:14 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5cecfc0137
commit ac2ae8798c
16 changed files with 669 additions and 73 deletions
@@ -36,6 +36,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatMessage,
)
from vllm.entrypoints.openai.engine.protocol import (
CompletionTokenUsageInfo,
DeltaMessage,
ErrorResponse,
FunctionCall,
@@ -105,6 +106,12 @@ def _make_prompt_tokens_details(
)
def _make_completion_tokens_details(
reasoning_tokens: int,
) -> CompletionTokenUsageInfo:
return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens)
class OpenAIServingChat(GenerateBaseServing):
def __init__(
self,
@@ -146,6 +153,7 @@ class OpenAIServingChat(GenerateBaseServing):
self.enable_log_deltas = enable_log_deltas
self.enable_auto_tools: bool = enable_auto_tools
self._include_reasoning_tokens_details = bool(reasoning_parser)
self.parser_cls = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
@@ -439,6 +447,9 @@ class OpenAIServingChat(GenerateBaseServing):
# Send response for each token for each request.n (index)
num_choices = 1 if request.n is None else request.n
previous_num_tokens = [0] * num_choices
# TODO: Remove once all reasoning parsers use the Parser Engine.
generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)]
previous_reasoning_tokens = [0] * num_choices
finish_reason_sent = [False] * num_choices
num_prompt_tokens = 0
num_cached_tokens = None
@@ -537,6 +548,11 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=0,
total_tokens=num_prompt_tokens,
completion_tokens_details=(
_make_completion_tokens_details(0)
if self._include_reasoning_tokens_details
else None
),
)
data = chunk.model_dump_json(exclude_unset=True)
@@ -573,6 +589,11 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=0,
total_tokens=num_prompt_tokens,
completion_tokens_details=(
_make_completion_tokens_details(0)
if self._include_reasoning_tokens_details
else None
),
)
data = chunk.model_dump_json(exclude_unset=True)
@@ -631,6 +652,11 @@ class OpenAIServingChat(GenerateBaseServing):
# set the previous values for the next iteration
previous_num_tokens[i] += len(output.token_ids)
if parser is not None:
generated_token_ids[i].extend(output.token_ids)
previous_reasoning_tokens[i] = parser.count_reasoning_tokens(
tuple(generated_token_ids[i])
)
# if the message delta is None (e.g. because it was a
# "control token" for tool calls or the parser otherwise
@@ -756,6 +782,13 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
completion_tokens_details=(
_make_completion_tokens_details(
previous_reasoning_tokens[i]
)
if self._include_reasoning_tokens_details
else None
),
)
data = chunk.model_dump_json(exclude_unset=True)
@@ -769,6 +802,11 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
completion_tokens_details=_make_completion_tokens_details(
sum(previous_reasoning_tokens)
)
if self._include_reasoning_tokens_details
else None,
)
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
@@ -813,6 +851,11 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=num_completion_tokens,
total_tokens=num_prompt_tokens + num_completion_tokens,
completion_tokens_details=_make_completion_tokens_details(
sum(previous_reasoning_tokens)
)
if self._include_reasoning_tokens_details
else None,
)
# Log complete streaming response if output logging is enabled
@@ -871,6 +914,7 @@ class OpenAIServingChat(GenerateBaseServing):
)
choices: list[ChatCompletionResponseChoice] = []
total_reasoning_tokens = 0
role = self.get_chat_request_role(request)
tool_parser_cls = (
@@ -908,6 +952,7 @@ class OpenAIServingChat(GenerateBaseServing):
suppress_metadata = not request.include_reasoning and parser is not None
if not request.include_reasoning:
reasoning = None
total_reasoning_tokens += parser.count_reasoning_tokens(token_ids)
if suppress_metadata:
logprobs = None
else:
@@ -1058,6 +1103,11 @@ class OpenAIServingChat(GenerateBaseServing):
prompt_tokens=num_prompt_tokens,
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
completion_tokens_details=_make_completion_tokens_details(
total_reasoning_tokens
)
if self._include_reasoning_tokens_details
else None,
)
usage.prompt_tokens_details = _make_prompt_tokens_details(
self.enable_prompt_tokens_details,
@@ -117,11 +117,16 @@ class PromptTokenUsageInfo(OpenAIBaseModel):
request has no multimodal input."""
class CompletionTokenUsageInfo(OpenAIBaseModel):
reasoning_tokens: int = 0
class UsageInfo(OpenAIBaseModel):
prompt_tokens: int = 0
total_tokens: int = 0
completion_tokens: int | None = 0
prompt_tokens_details: PromptTokenUsageInfo | None = None
completion_tokens_details: CompletionTokenUsageInfo | None = None
class PerRequestTimingMetrics(OpenAIBaseModel):
+6 -8
View File
@@ -487,6 +487,7 @@ class OpenAIServingResponses(GenerateBaseServing):
response_parser=response_parser,
)
reasoning_parser_kwargs = None
if (
context.response_parser is not None
and context.response_parser.reasoning_parser is not None
@@ -518,9 +519,7 @@ class OpenAIServingResponses(GenerateBaseServing):
priority=self._get_priority(request, raw_request),
trace_headers=trace_headers,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs
if self.parser and self.parser.reasoning_parser_cls is not None
else None,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
generators.append(generator)
@@ -873,6 +872,8 @@ class OpenAIServingResponses(GenerateBaseServing):
if final_output.finish_reason == "length":
status = "incomplete"
# TODO: Build final response items from the accumulated streaming
# parser results instead of reparsing the complete output.
output = self._make_response_output_items(
request,
final_output,
@@ -901,13 +902,10 @@ class OpenAIServingResponses(GenerateBaseServing):
num_reasoning_tokens == 0
and isinstance(context, (SimpleContext, ParsableContext))
and context.response_parser is not None
and context.response_parser.reasoning_parser is not None
):
accumulated = getattr(context, "_accumulated_token_ids", []) or []
num_reasoning_tokens = (
context.response_parser.reasoning_parser.count_reasoning_tokens(
accumulated
)
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
accumulated
)
usage = ResponseUsage(