Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -207,6 +207,11 @@ if _SDK_AVAILABLE:
|
||||
if isinstance(result, ErrorResponse):
|
||||
return _error_response(result, raw_request)
|
||||
|
||||
if (kwargs := result.reasoning_parser_kwargs) is not None:
|
||||
kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
|
||||
kwargs.chat_template_kwargs
|
||||
)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Adapted from
|
||||
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
||||
import time
|
||||
from typing import Annotated, Any, ClassVar, Literal
|
||||
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
|
||||
|
||||
from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
@@ -53,6 +53,9 @@ from vllm.sampling_params import (
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -566,6 +569,25 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
_grammar_from_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
|
||||
|
||||
def resolve_reasoning_ended(
|
||||
self, parser: "Parser | None", prompt_token_ids: list[int]
|
||||
) -> bool | None:
|
||||
"""Resolve the engine's `reasoning_ended` for structured outputs.
|
||||
|
||||
Call after `adjust_request`, which may set `_grammar_from_parser`.
|
||||
`True` constrains from the first generated token; `None` lets the
|
||||
engine check the prompt with its own reasoning parser.
|
||||
"""
|
||||
if not self.include_reasoning:
|
||||
return True
|
||||
if self._grammar_from_parser:
|
||||
# The Mistral grammar already includes an optional `think?`
|
||||
# rule that handles both reasoning and non-reasoning outputs.
|
||||
return True
|
||||
if parser is not None and parser.reasoning_parser is not None:
|
||||
return parser.is_reasoning_end(prompt_token_ids)
|
||||
return None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
|
||||
@@ -364,17 +364,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
session_id=session_id,
|
||||
)
|
||||
else:
|
||||
if not request.include_reasoning:
|
||||
reasoning_ended = True
|
||||
elif request._grammar_from_parser:
|
||||
# The Mistral grammar already includes an optional
|
||||
# `think?` rule that handles both reasoning and
|
||||
# non-reasoning outputs.
|
||||
reasoning_ended = True
|
||||
elif parser is not None and parser.reasoning_parser is not None:
|
||||
reasoning_ended = parser.is_reasoning_end(prompt_token_ids or [])
|
||||
else:
|
||||
reasoning_ended = None
|
||||
reasoning_ended = request.resolve_reasoning_ended(
|
||||
parser, prompt_token_ids or []
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
@@ -124,6 +125,20 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
# Resolve here what OpenAIServingChat passes to the engine:
|
||||
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
|
||||
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
|
||||
parser = None
|
||||
if reasoning_parser_kwargs is not None:
|
||||
assert self.online_renderer.parser is not None
|
||||
parser = self.online_renderer.parser(
|
||||
self.online_renderer.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
@@ -132,6 +147,8 @@ class ServingRender(BaseServing):
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
@@ -263,6 +280,7 @@ class ServingRender(BaseServing):
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
@@ -271,6 +289,18 @@ class ServingRender(BaseServing):
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
def _reasoning_parser_kwargs(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ReasoningParserKwargs | None:
|
||||
parser_cls = self.online_renderer.parser
|
||||
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
|
||||
return None
|
||||
return ReasoningParserKwargs(
|
||||
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
|
||||
request
|
||||
)
|
||||
)
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||
if self._placeholder_metadata_parser_failed:
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Discriminator,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
@@ -147,6 +148,16 @@ class MultiModalFeatures(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class ReasoningParserKwargs(BaseModel):
|
||||
"""Kwargs for the engine-side reasoning parser that gates structured
|
||||
outputs. Typed so clients cannot pass arbitrary constructor kwargs."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)
|
||||
"""The effective chat template kwargs the prompt was rendered with."""
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
@@ -212,6 +223,15 @@ class GenerateRequest(BaseModel):
|
||||
|
||||
model: str | None = None
|
||||
|
||||
reasoning_ended: bool | None = None
|
||||
"""Whether reasoning has ended before the first generated token, as
|
||||
resolved by /render. `True` applies structured outputs from the first
|
||||
token; `None` lets the engine check the prompt with its reasoning parser."""
|
||||
|
||||
reasoning_parser_kwargs: ReasoningParserKwargs | None = None
|
||||
"""Set by /render when it has a reasoning parser, so the engine-side
|
||||
parser agrees with the frontend on flags such as `enable_thinking`."""
|
||||
|
||||
return_token_ids: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -349,6 +349,12 @@ class ServingTokens(GenerateBaseServing):
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
session_id=session_id,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=(
|
||||
request.reasoning_parser_kwargs.model_dump()
|
||||
if request.reasoning_parser_kwargs is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
Reference in New Issue
Block a user