[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5cecfc0137
commit
ac2ae8798c
@@ -36,6 +36,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
CompletionTokenUsageInfo,
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
@@ -105,6 +106,12 @@ def _make_prompt_tokens_details(
|
||||
)
|
||||
|
||||
|
||||
def _make_completion_tokens_details(
|
||||
reasoning_tokens: int,
|
||||
) -> CompletionTokenUsageInfo:
|
||||
return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens)
|
||||
|
||||
|
||||
class OpenAIServingChat(GenerateBaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -146,6 +153,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
self.enable_log_deltas = enable_log_deltas
|
||||
|
||||
self.enable_auto_tools: bool = enable_auto_tools
|
||||
self._include_reasoning_tokens_details = bool(reasoning_parser)
|
||||
self.parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
@@ -439,6 +447,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# Send response for each token for each request.n (index)
|
||||
num_choices = 1 if request.n is None else request.n
|
||||
previous_num_tokens = [0] * num_choices
|
||||
# TODO: Remove once all reasoning parsers use the Parser Engine.
|
||||
generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)]
|
||||
previous_reasoning_tokens = [0] * num_choices
|
||||
finish_reason_sent = [False] * num_choices
|
||||
num_prompt_tokens = 0
|
||||
num_cached_tokens = None
|
||||
@@ -537,6 +548,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=0,
|
||||
total_tokens=num_prompt_tokens,
|
||||
completion_tokens_details=(
|
||||
_make_completion_tokens_details(0)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
@@ -573,6 +589,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=0,
|
||||
total_tokens=num_prompt_tokens,
|
||||
completion_tokens_details=(
|
||||
_make_completion_tokens_details(0)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
@@ -631,6 +652,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
# set the previous values for the next iteration
|
||||
previous_num_tokens[i] += len(output.token_ids)
|
||||
if parser is not None:
|
||||
generated_token_ids[i].extend(output.token_ids)
|
||||
previous_reasoning_tokens[i] = parser.count_reasoning_tokens(
|
||||
tuple(generated_token_ids[i])
|
||||
)
|
||||
|
||||
# if the message delta is None (e.g. because it was a
|
||||
# "control token" for tool calls or the parser otherwise
|
||||
@@ -756,6 +782,13 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
completion_tokens_details=(
|
||||
_make_completion_tokens_details(
|
||||
previous_reasoning_tokens[i]
|
||||
)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
@@ -769,6 +802,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
completion_tokens_details=_make_completion_tokens_details(
|
||||
sum(previous_reasoning_tokens)
|
||||
)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None,
|
||||
)
|
||||
final_usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||
self.enable_prompt_tokens_details,
|
||||
@@ -813,6 +851,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_completion_tokens,
|
||||
total_tokens=num_prompt_tokens + num_completion_tokens,
|
||||
completion_tokens_details=_make_completion_tokens_details(
|
||||
sum(previous_reasoning_tokens)
|
||||
)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None,
|
||||
)
|
||||
|
||||
# Log complete streaming response if output logging is enabled
|
||||
@@ -871,6 +914,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
total_reasoning_tokens = 0
|
||||
|
||||
role = self.get_chat_request_role(request)
|
||||
tool_parser_cls = (
|
||||
@@ -908,6 +952,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
suppress_metadata = not request.include_reasoning and parser is not None
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
total_reasoning_tokens += parser.count_reasoning_tokens(token_ids)
|
||||
if suppress_metadata:
|
||||
logprobs = None
|
||||
else:
|
||||
@@ -1058,6 +1103,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
completion_tokens_details=_make_completion_tokens_details(
|
||||
total_reasoning_tokens
|
||||
)
|
||||
if self._include_reasoning_tokens_details
|
||||
else None,
|
||||
)
|
||||
usage.prompt_tokens_details = _make_prompt_tokens_details(
|
||||
self.enable_prompt_tokens_details,
|
||||
|
||||
@@ -117,11 +117,16 @@ class PromptTokenUsageInfo(OpenAIBaseModel):
|
||||
request has no multimodal input."""
|
||||
|
||||
|
||||
class CompletionTokenUsageInfo(OpenAIBaseModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class UsageInfo(OpenAIBaseModel):
|
||||
prompt_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
completion_tokens: int | None = 0
|
||||
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
||||
completion_tokens_details: CompletionTokenUsageInfo | None = None
|
||||
|
||||
|
||||
class PerRequestTimingMetrics(OpenAIBaseModel):
|
||||
|
||||
@@ -487,6 +487,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
response_parser=response_parser,
|
||||
)
|
||||
|
||||
reasoning_parser_kwargs = None
|
||||
if (
|
||||
context.response_parser is not None
|
||||
and context.response_parser.reasoning_parser is not None
|
||||
@@ -518,9 +519,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
priority=self._get_priority(request, raw_request),
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None
|
||||
else None,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
@@ -873,6 +872,8 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if final_output.finish_reason == "length":
|
||||
status = "incomplete"
|
||||
|
||||
# TODO: Build final response items from the accumulated streaming
|
||||
# parser results instead of reparsing the complete output.
|
||||
output = self._make_response_output_items(
|
||||
request,
|
||||
final_output,
|
||||
@@ -901,13 +902,10 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
num_reasoning_tokens == 0
|
||||
and isinstance(context, (SimpleContext, ParsableContext))
|
||||
and context.response_parser is not None
|
||||
and context.response_parser.reasoning_parser is not None
|
||||
):
|
||||
accumulated = getattr(context, "_accumulated_token_ids", []) or []
|
||||
num_reasoning_tokens = (
|
||||
context.response_parser.reasoning_parser.count_reasoning_tokens(
|
||||
accumulated
|
||||
)
|
||||
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
|
||||
accumulated
|
||||
)
|
||||
|
||||
usage = ResponseUsage(
|
||||
|
||||
Reference in New Issue
Block a user