[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -110,7 +110,7 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.utils import get_max_tokens
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import ProcessorInputs, token_inputs
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
@@ -269,10 +269,10 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
|
||||
def _validate_generator_input(
|
||||
self,
|
||||
engine_prompt: ProcessorInputs,
|
||||
engine_input: EngineInput,
|
||||
) -> ErrorResponse | None:
|
||||
"""Add validations to the input to the generator here."""
|
||||
prompt_len = self._extract_prompt_len(engine_prompt)
|
||||
prompt_len = self._extract_prompt_len(engine_input)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
if prompt_len >= max_model_len:
|
||||
@@ -369,11 +369,11 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
model_name = self.models.model_name(lora_request)
|
||||
|
||||
if self.use_harmony:
|
||||
messages, engine_prompts = self._make_request_with_harmony(
|
||||
messages, engine_inputs = self._make_request_with_harmony(
|
||||
request, prev_response
|
||||
)
|
||||
else:
|
||||
messages, engine_prompts = await self._make_request(request, prev_response)
|
||||
messages, engine_inputs = await self._make_request(request, prev_response)
|
||||
|
||||
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||
if raw_request:
|
||||
@@ -413,15 +413,15 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
available_tools = []
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
|
||||
for engine_prompt in engine_prompts:
|
||||
maybe_error = self._validate_generator_input(engine_prompt)
|
||||
for engine_input in engine_inputs:
|
||||
maybe_error = self._validate_generator_input(engine_input)
|
||||
if maybe_error is not None:
|
||||
return maybe_error
|
||||
|
||||
default_max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
request.max_output_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self._extract_prompt_len(engine_input),
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
@@ -480,7 +480,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
)
|
||||
generator = self._generate_with_builtin_tools(
|
||||
request_id=request.request_id,
|
||||
engine_prompt=engine_prompt,
|
||||
engine_input=engine_input,
|
||||
sampling_params=sampling_params,
|
||||
context=context,
|
||||
lora_request=lora_request,
|
||||
@@ -586,7 +586,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
prev_response_output=prev_response.output if prev_response else None,
|
||||
)
|
||||
|
||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
request,
|
||||
messages,
|
||||
default_template=self.chat_template,
|
||||
@@ -595,7 +595,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
||||
)
|
||||
return messages, engine_prompts
|
||||
return messages, engine_inputs
|
||||
|
||||
async def _render_next_turn(
|
||||
self,
|
||||
@@ -610,7 +610,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
request_input=messages,
|
||||
)
|
||||
|
||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
request,
|
||||
new_messages,
|
||||
default_template=chat_template,
|
||||
@@ -619,12 +619,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
)
|
||||
return engine_prompts
|
||||
return engine_inputs
|
||||
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: ProcessorInputs,
|
||||
engine_input: EngineInput,
|
||||
sampling_params: SamplingParams,
|
||||
context: ConversationContext,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -641,13 +641,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
|
||||
self._log_inputs(
|
||||
sub_request_id,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
@@ -675,11 +675,11 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
# Render the next prompt token ids and update sampling_params.
|
||||
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
||||
token_ids = context.render_for_completion()
|
||||
engine_prompt = token_inputs(token_ids)
|
||||
engine_input = tokens_input(token_ids)
|
||||
|
||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
||||
elif isinstance(context, ParsableContext):
|
||||
(engine_prompt,) = await self._render_next_turn(
|
||||
(engine_input,) = await self._render_next_turn(
|
||||
context.request,
|
||||
context.parser.response_messages,
|
||||
context.tool_dicts,
|
||||
@@ -691,7 +691,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
context.request.max_output_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self._extract_prompt_len(engine_input),
|
||||
self.default_sampling_params, # type: ignore
|
||||
self.override_max_tokens, # type: ignore
|
||||
)
|
||||
@@ -713,14 +713,10 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
arrival_time = time.time()
|
||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
||||
prompt_token_ids = render_for_completion(messages)
|
||||
engine_prompt = token_inputs(prompt_token_ids)
|
||||
engine_prompt["arrival_time"] = arrival_time
|
||||
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
|
||||
# Add cache_salt if provided in the request
|
||||
if request.cache_salt is not None:
|
||||
engine_prompt["cache_salt"] = request.cache_salt
|
||||
|
||||
return messages, [engine_prompt]
|
||||
return messages, [engine_input]
|
||||
|
||||
async def _initialize_tool_sessions(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user