[Frontend] Complete OpenAI render delegation (#37287)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -29,6 +29,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import RequestOutput
|
||||
@@ -45,6 +46,7 @@ class ServingTokens(OpenAIServing):
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
openai_serving_render: OpenAIServingRender,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
force_no_detokenize: bool = False,
|
||||
@@ -58,6 +60,7 @@ class ServingTokens(OpenAIServing):
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
self.force_no_detokenize = force_no_detokenize
|
||||
@@ -96,7 +99,7 @@ class ServingTokens(OpenAIServing):
|
||||
if raw_request:
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
engine_prompts = await self._preprocess_completion(
|
||||
engine_prompts = await self.openai_serving_render.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.token_ids,
|
||||
prompt_embeds=None,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
parse_chat_inputs_to_harmony_messages,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
@@ -459,9 +460,9 @@ class OpenAIServingRender:
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
return await self._preprocess_cmpl(request, prompts)
|
||||
return await self.preprocess_cmpl(request, prompts)
|
||||
|
||||
async def _preprocess_cmpl(
|
||||
async def preprocess_cmpl(
|
||||
self,
|
||||
request: Any,
|
||||
prompts: Sequence[PromptType | bytes],
|
||||
@@ -500,11 +501,7 @@ class OpenAIServingRender:
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
|
||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
|
||||
"""Copied from OpenAIServing._preprocess_chat.
|
||||
|
||||
Differences: isinstance check is ChatCompletionRequest-only
|
||||
(ResponsesRequest not supported here); TODO comment dropped accordingly.
|
||||
"""
|
||||
"""Copied from OpenAIServing._preprocess_chat."""
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
@@ -542,11 +539,11 @@ class OpenAIServingRender:
|
||||
if tool_parser is not None:
|
||||
tool_choice = getattr(request, "tool_choice", "none")
|
||||
if tool_choice != "none":
|
||||
if not isinstance(request, ChatCompletionRequest):
|
||||
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
|
||||
msg = (
|
||||
"Tool usage is only supported "
|
||||
" for ChatCompletionRequest, but got "
|
||||
f"{type(request).__name__}"
|
||||
"for Chat Completions API or Responses API requests, "
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
|
||||
Reference in New Issue
Block a user