Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
# Adapted from
|
||||
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||
import time
|
||||
from typing import Annotated, Any, ClassVar, Literal
|
||||
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
|
||||
|
||||
from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
@@ -53,6 +53,9 @@ from vllm.sampling_params import (
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -566,6 +569,25 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
_grammar_from_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
|
||||
|
||||
def resolve_reasoning_ended(
|
||||
self, parser: "Parser | None", prompt_token_ids: list[int]
|
||||
) -> bool | None:
|
||||
"""Resolve the engine's `reasoning_ended` for structured outputs.
|
||||
|
||||
Call after `adjust_request`, which may set `_grammar_from_parser`.
|
||||
`True` constrains from the first generated token; `None` lets the
|
||||
engine check the prompt with its own reasoning parser.
|
||||
"""
|
||||
if not self.include_reasoning:
|
||||
return True
|
||||
if self._grammar_from_parser:
|
||||
# The Mistral grammar already includes an optional `think?`
|
||||
# rule that handles both reasoning and non-reasoning outputs.
|
||||
return True
|
||||
if parser is not None and parser.reasoning_parser is not None:
|
||||
return parser.is_reasoning_end(prompt_token_ids)
|
||||
return None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
|
||||
@@ -364,17 +364,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
session_id=session_id,
|
||||
)
|
||||
else:
|
||||
if not request.include_reasoning:
|
||||
reasoning_ended = True
|
||||
elif request._grammar_from_parser:
|
||||
# The Mistral grammar already includes an optional
|
||||
# `think?` rule that handles both reasoning and
|
||||
# non-reasoning outputs.
|
||||
reasoning_ended = True
|
||||
elif parser is not None and parser.reasoning_parser is not None:
|
||||
reasoning_ended = parser.is_reasoning_end(prompt_token_ids or [])
|
||||
else:
|
||||
reasoning_ended = None
|
||||
reasoning_ended = request.resolve_reasoning_ended(
|
||||
parser, prompt_token_ids or []
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
|
||||
Reference in New Issue
Block a user