Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
@@ -124,6 +125,20 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
# Resolve here what OpenAIServingChat passes to the engine:
|
||||
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
|
||||
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
|
||||
parser = None
|
||||
if reasoning_parser_kwargs is not None:
|
||||
assert self.online_renderer.parser is not None
|
||||
parser = self.online_renderer.parser(
|
||||
self.online_renderer.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
@@ -132,6 +147,8 @@ class ServingRender(BaseServing):
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
@@ -263,6 +280,7 @@ class ServingRender(BaseServing):
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
@@ -271,6 +289,18 @@ class ServingRender(BaseServing):
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
def _reasoning_parser_kwargs(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ReasoningParserKwargs | None:
|
||||
parser_cls = self.online_renderer.parser
|
||||
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
|
||||
return None
|
||||
return ReasoningParserKwargs(
|
||||
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
|
||||
request
|
||||
)
|
||||
)
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||
if self._placeholder_metadata_parser_failed:
|
||||
|
||||
Reference in New Issue
Block a user