[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -33,7 +33,7 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import ProcessorInputs
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import RequestOutput
|
||||
@@ -82,7 +82,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
) -> list[ProcessorInputs] | ErrorResponse:
|
||||
) -> list[EngineInput] | ErrorResponse:
|
||||
"""
|
||||
Validate the model and preprocess a completion request.
|
||||
|
||||
@@ -90,8 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
|
||||
Returns:
|
||||
A list of engine_prompts on success,
|
||||
or an ErrorResponse on failure.
|
||||
A list of engine_inputs on success, or an ErrorResponse on failure.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
@@ -128,7 +127,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
engine_prompts = result
|
||||
engine_inputs = result
|
||||
|
||||
request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
created_time = int(time.time())
|
||||
@@ -145,11 +144,11 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
request.max_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self._extract_prompt_len(engine_input),
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
@@ -169,7 +168,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
@@ -182,7 +181,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_prompt,
|
||||
prompt=engine_input,
|
||||
request_id=request_id,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
@@ -190,7 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
)
|
||||
else:
|
||||
generator = self.engine_client.generate(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -204,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
result_generator = merge_async_iterators(*generators)
|
||||
|
||||
model_name = self.models.model_name(lora_request)
|
||||
num_prompts = len(engine_prompts)
|
||||
num_prompts = len(engine_inputs)
|
||||
|
||||
# Streaming response
|
||||
tokenizer = self.renderer.tokenizer
|
||||
@@ -212,7 +211,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if request.stream:
|
||||
return self.completion_stream_generator(
|
||||
request,
|
||||
engine_prompts,
|
||||
engine_inputs,
|
||||
result_generator,
|
||||
request_id,
|
||||
created_time,
|
||||
@@ -235,8 +234,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
# We did not pass it into vLLM engine to avoid being redundant
|
||||
# with the inputs token IDs
|
||||
if final_res.prompt is None:
|
||||
engine_prompt = engine_prompts[i]
|
||||
final_res.prompt = self._extract_prompt_text(engine_prompt)
|
||||
final_res.prompt = self._extract_prompt_text(engine_inputs[i])
|
||||
|
||||
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
|
||||
|
||||
@@ -268,7 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def completion_stream_generator(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
engine_prompts: list[ProcessorInputs],
|
||||
engine_inputs: list[EngineInput],
|
||||
result_generator: AsyncIterator[tuple[int, RequestOutput]],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
@@ -301,8 +299,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
|
||||
prompt_text = res.prompt
|
||||
if prompt_text is None:
|
||||
engine_prompt = engine_prompts[prompt_idx]
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
engine_input = engine_inputs[prompt_idx]
|
||||
prompt_text = self._extract_prompt_text(engine_input)
|
||||
|
||||
# Prompt details are excluded from later streamed outputs
|
||||
if prompt_token_ids is not None:
|
||||
|
||||
Reference in New Issue
Block a user