[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -63,7 +63,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.inputs.data import ProcessorInputs
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
@@ -177,7 +177,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||
"""
|
||||
Validate the model and preprocess a chat completion request.
|
||||
|
||||
@@ -185,7 +185,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
|
||||
Returns:
|
||||
A tuple of (conversation, engine_prompts) on success,
|
||||
A tuple of (conversation, engine_inputs) on success,
|
||||
or an ErrorResponse on failure.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
@@ -231,7 +231,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
conversation, engine_prompts = result
|
||||
conversation, engine_inputs = result
|
||||
|
||||
request_id = (
|
||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
@@ -251,13 +251,13 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
|
||||
|
||||
# If we are creating sub requests for multiple prompts, ensure that they
|
||||
# have unique request ids.
|
||||
sub_request_id = (
|
||||
request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
|
||||
request_id if len(engine_inputs) == 1 else f"{request_id}_{i}"
|
||||
)
|
||||
|
||||
max_tokens = get_max_tokens(
|
||||
@@ -265,7 +265,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self._extract_prompt_len(engine_input),
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
@@ -283,7 +283,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
self._log_inputs(
|
||||
sub_request_id,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
@@ -296,7 +296,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_prompt,
|
||||
prompt=engine_input,
|
||||
request_id=sub_request_id,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
@@ -313,7 +313,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
reasoning_ended = None
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
|
||||
Reference in New Issue
Block a user