[Frontend] Use new Renderer for Completions and Tokenize API (#32863)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -114,16 +114,15 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
construct_input_messages,
|
||||
construct_tool_dicts,
|
||||
extract_tool_types,
|
||||
should_continue_final_message,
|
||||
)
|
||||
from vllm.entrypoints.utils import get_max_tokens
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import TokensPrompt
|
||||
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
|
||||
from vllm.inputs.parse import get_prompt_len
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.renderers import RendererLike
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
@@ -291,13 +290,14 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self.tool_server = tool_server
|
||||
|
||||
def _validate_generator_input(
|
||||
self, engine_prompt: TokensPrompt
|
||||
self,
|
||||
engine_prompt: TokensPrompt | EmbedsPrompt,
|
||||
) -> ErrorResponse | None:
|
||||
"""Add validations to the input to the generator here."""
|
||||
if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
|
||||
prompt_len = get_prompt_len(engine_prompt)
|
||||
if self.max_model_len <= prompt_len:
|
||||
error_message = (
|
||||
"The engine prompt length"
|
||||
f" {len(engine_prompt['prompt_token_ids'])} "
|
||||
f"The engine prompt length {prompt_len} "
|
||||
f"exceeds the max_model_len {self.max_model_len}. "
|
||||
"Please reduce prompt."
|
||||
)
|
||||
@@ -307,6 +307,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
param="input",
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _validate_create_responses_input(
|
||||
@@ -387,8 +388,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
try:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
renderer = self.engine_client.renderer
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
|
||||
if self.use_harmony:
|
||||
messages, engine_prompts = self._make_request_with_harmony(
|
||||
@@ -396,7 +395,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
)
|
||||
else:
|
||||
messages, engine_prompts = await self._make_request(
|
||||
request, prev_response, renderer
|
||||
request, prev_response
|
||||
)
|
||||
|
||||
except (
|
||||
@@ -431,6 +430,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
assert len(builtin_tool_list) == 0
|
||||
available_tools = []
|
||||
try:
|
||||
renderer = self.engine_client.renderer
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
|
||||
for engine_prompt in engine_prompts:
|
||||
maybe_error = self._validate_generator_input(engine_prompt)
|
||||
if maybe_error is not None:
|
||||
@@ -446,6 +448,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
sampling_params = request.to_sampling_params(
|
||||
default_max_tokens, self.default_sampling_params
|
||||
)
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
@@ -465,7 +468,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
# tokens during generation instead of at the end
|
||||
context = ParsableContext(
|
||||
response_messages=messages,
|
||||
renderer=renderer,
|
||||
tokenizer=tokenizer,
|
||||
reasoning_parser_cls=self.reasoning_parser,
|
||||
request=request,
|
||||
tool_parser_cls=self.tool_parser,
|
||||
@@ -495,6 +498,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
request_id=request.request_id,
|
||||
engine_prompt=engine_prompt,
|
||||
sampling_params=sampling_params,
|
||||
tok_params=tok_params,
|
||||
context=context,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
@@ -596,7 +600,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
renderer: RendererLike,
|
||||
):
|
||||
tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
|
||||
# Construct the input messages.
|
||||
@@ -606,30 +609,15 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
|
||||
prev_response_output=prev_response.output if prev_response else None,
|
||||
)
|
||||
# Check if we should continue the final message (partial completion)
|
||||
# This enables Anthropic-style partial message completion where the
|
||||
# user provides an incomplete assistant message to continue from.
|
||||
continue_final = should_continue_final_message(request.input)
|
||||
chat_template_kwargs = dict(
|
||||
reasoning_effort=None
|
||||
if request.reasoning is None
|
||||
else request.reasoning.effort
|
||||
)
|
||||
|
||||
_, engine_prompts = await self._preprocess_chat(
|
||||
request,
|
||||
renderer,
|
||||
messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=None,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.tool_parser,
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
# When continuing a partial message, we set continue_final_message=True
|
||||
# and add_generation_prompt=False so the model continues the message
|
||||
# rather than starting a new one.
|
||||
add_generation_prompt=not continue_final,
|
||||
continue_final_message=continue_final,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
return messages, engine_prompts
|
||||
|
||||
|
||||
Reference in New Issue
Block a user