[Misc] Reorganize inputs (#35182)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-25 10:22:54 -07:00
committed by GitHub
parent 678b3c99e8
commit ba2f0acc2d
142 changed files with 1212 additions and 1342 deletions
+15 -17
View File
@@ -33,7 +33,7 @@ from vllm.entrypoints.openai.engine.serving import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
from vllm.exceptions import VLLMValidationError
from vllm.inputs.data import ProcessorInputs
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import RequestOutput
@@ -82,7 +82,7 @@ class OpenAIServingCompletion(OpenAIServing):
async def render_completion_request(
self,
request: CompletionRequest,
) -> list[ProcessorInputs] | ErrorResponse:
) -> list[EngineInput] | ErrorResponse:
"""
Validate the model and preprocess a completion request.
@@ -90,8 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):
engine-aware checks (LoRA model validation, engine health).
Returns:
A list of engine_prompts on success,
or an ErrorResponse on failure.
A list of engine_inputs on success, or an ErrorResponse on failure.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
@@ -128,7 +127,7 @@ class OpenAIServingCompletion(OpenAIServing):
if isinstance(result, ErrorResponse):
return result
engine_prompts = result
engine_inputs = result
request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
created_time = int(time.time())
@@ -145,11 +144,11 @@ class OpenAIServingCompletion(OpenAIServing):
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = []
for i, engine_prompt in enumerate(engine_prompts):
for i, engine_input in enumerate(engine_inputs):
max_tokens = get_max_tokens(
max_model_len,
request.max_tokens,
self._extract_prompt_len(engine_prompt),
self._extract_prompt_len(engine_input),
self.default_sampling_params,
self.override_max_tokens,
)
@@ -169,7 +168,7 @@ class OpenAIServingCompletion(OpenAIServing):
self._log_inputs(
request_id_item,
engine_prompt,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
@@ -182,7 +181,7 @@ class OpenAIServingCompletion(OpenAIServing):
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
prompt=engine_input,
request_id=request_id,
params=sampling_params,
lora_request=lora_request,
@@ -190,7 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):
)
else:
generator = self.engine_client.generate(
engine_prompt,
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
@@ -204,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
result_generator = merge_async_iterators(*generators)
model_name = self.models.model_name(lora_request)
num_prompts = len(engine_prompts)
num_prompts = len(engine_inputs)
# Streaming response
tokenizer = self.renderer.tokenizer
@@ -212,7 +211,7 @@ class OpenAIServingCompletion(OpenAIServing):
if request.stream:
return self.completion_stream_generator(
request,
engine_prompts,
engine_inputs,
result_generator,
request_id,
created_time,
@@ -235,8 +234,7 @@ class OpenAIServingCompletion(OpenAIServing):
# We did not pass it into vLLM engine to avoid being redundant
# with the inputs token IDs
if final_res.prompt is None:
engine_prompt = engine_prompts[i]
final_res.prompt = self._extract_prompt_text(engine_prompt)
final_res.prompt = self._extract_prompt_text(engine_inputs[i])
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
@@ -268,7 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
async def completion_stream_generator(
self,
request: CompletionRequest,
engine_prompts: list[ProcessorInputs],
engine_inputs: list[EngineInput],
result_generator: AsyncIterator[tuple[int, RequestOutput]],
request_id: str,
created_time: int,
@@ -301,8 +299,8 @@ class OpenAIServingCompletion(OpenAIServing):
prompt_text = res.prompt
if prompt_text is None:
engine_prompt = engine_prompts[prompt_idx]
prompt_text = self._extract_prompt_text(engine_prompt)
engine_input = engine_inputs[prompt_idx]
prompt_text = self._extract_prompt_text(engine_input)
# Prompt details are excluded from later streamed outputs
if prompt_token_ids is not None: