[Misc] Reorganize inputs (#35182)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-25 10:22:54 -07:00
committed by GitHub
parent 678b3c99e8
commit ba2f0acc2d
142 changed files with 1212 additions and 1342 deletions
+10 -10
View File
@@ -18,7 +18,7 @@ from vllm.entrypoints.pooling.typing import (
PoolingCompletionLikeRequest,
PoolingServeContext,
)
from vllm.inputs.data import ProcessorInputs, SingletonPrompt
from vllm.inputs import EngineInput, SingletonPrompt
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.tool_parsers import ToolParser
@@ -60,7 +60,7 @@ class PoolingIOProcessor:
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
_, engine_prompts = self._preprocess_chat_online(
_, engine_inputs = self._preprocess_chat_online(
request,
request.messages,
default_template=self.chat_template,
@@ -68,7 +68,7 @@ class PoolingIOProcessor:
default_template_kwargs=None,
)
elif isinstance(request, PoolingCompletionLikeRequest):
engine_prompts = self._preprocess_completion_online(
engine_inputs = self._preprocess_completion_online(
request,
prompt_input=request.input,
prompt_embeds=None,
@@ -76,7 +76,7 @@ class PoolingIOProcessor:
else:
raise ValueError(f"Invalid {self.name} request type")
ctx.engine_prompts = engine_prompts
ctx.engine_inputs = engine_inputs
async def pre_process_online_async(self, ctx: PoolingServeContext):
self.pre_process_online(ctx)
@@ -100,7 +100,7 @@ class PoolingIOProcessor:
self,
prompts: PromptType | Sequence[PromptType],
tokenization_kwargs: dict[str, Any] | None = None,
) -> Sequence[ProcessorInputs]:
) -> Sequence[EngineInput]:
return self._preprocess_completion_offline(
prompts=prompts, tokenization_kwargs=tokenization_kwargs
)
@@ -128,7 +128,7 @@ class PoolingIOProcessor:
request: RendererRequest,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
) -> list[ProcessorInputs]:
) -> list[EngineInput]:
renderer = self.renderer
model_config = self.model_config
@@ -167,7 +167,7 @@ class PoolingIOProcessor:
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
tool_parser: type[ToolParser] | None = None,
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
) -> tuple[list[ConversationMessage], list[EngineInput]]:
renderer = self.renderer
default_template_kwargs = merge_kwargs(
@@ -188,7 +188,7 @@ class PoolingIOProcessor:
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
(conversation,), (engine_prompt,) = renderer.render_chat(
(conversation,), (engine_input,) = renderer.render_chat(
[messages],
chat_params,
tok_params,
@@ -199,13 +199,13 @@ class PoolingIOProcessor:
},
)
return conversation, [engine_prompt]
return conversation, [engine_input]
def _preprocess_completion_offline(
self,
prompts: PromptType | Sequence[PromptType],
tokenization_kwargs: dict[str, Any] | None = None,
) -> Sequence[ProcessorInputs]:
) -> Sequence[EngineInput]:
renderer = self.renderer
model_config = self.model_config
+8 -8
View File
@@ -20,7 +20,7 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.pooling.typing import AnyPoolingRequest, PoolingServeContext
from vllm.exceptions import VLLMNotFoundError
from vllm.inputs.data import ProcessorInputs
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.base import BaseRenderer
from vllm.renderers.inputs.preprocess import extract_prompt_components
@@ -106,7 +106,7 @@ class PoolingServing:
self,
ctx: PoolingServeContext,
):
if ctx.engine_prompts is None:
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
@@ -120,7 +120,7 @@ class PoolingServing:
pooling_params = self.io_processor.create_pooling_params(ctx.request)
pooling_params.verify(self.model_config)
for i, engine_prompt in enumerate(ctx.engine_prompts):
for i, engine_input in enumerate(ctx.engine_inputs):
prompt_request_id = (
f"{ctx.request_id}-{i}"
if ctx.prompt_request_ids is None
@@ -129,13 +129,13 @@ class PoolingServing:
self._log_inputs(
prompt_request_id,
engine_prompt,
engine_input,
params=pooling_params,
lora_request=ctx.lora_request,
)
generator = self.engine_client.encode(
engine_prompt,
engine_input,
pooling_params,
prompt_request_id,
lora_request=ctx.lora_request,
@@ -151,13 +151,13 @@ class PoolingServing:
self,
ctx: PoolingServeContext,
):
if ctx.engine_prompts is None:
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
if ctx.result_generator is None:
raise ValueError("Result generator not available")
num_inputs = len(ctx.engine_prompts)
num_inputs = len(ctx.engine_inputs)
final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_inputs
@@ -317,7 +317,7 @@ class PoolingServing:
def _log_inputs(
self,
request_id: str,
inputs: ProcessorInputs,
inputs: EngineInput,
params: PoolingParams,
lora_request: LoRARequest | None,
) -> None:
+20 -20
View File
@@ -24,7 +24,7 @@ from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingCompletionRequest,
)
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.inputs.data import ProcessorInputs, token_inputs
from vllm.inputs import EngineInput, tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
from vllm.renderers import merge_kwargs
@@ -83,20 +83,20 @@ class EmbedIOProcessor(PoolingIOProcessor):
#################################################################
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
if ctx.engine_prompts is None:
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
ctx.intermediates = ctx.engine_prompts
ctx.intermediates = ctx.engine_inputs
request_id = ctx.request_id
max_model_len = self.model_config.max_model_len
chunked_engine_prompts: list[ProcessorInputs] = []
chunked_engine_inputs: list[EngineInput] = []
prompt_request_ids: list[str] = []
for prompt_idx, engine_prompt in enumerate(ctx.engine_prompts):
token_ids = engine_prompt.get("prompt_token_ids", None)
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
token_ids = engine_input.get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInputs."
"not support EmbedsPrompt and EncoderDecoderInput."
)
prompt_token_ids = cast(list[int], token_ids)
@@ -104,14 +104,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
for chunk_idx, chunk_tokens in enumerate(
chunk_list(prompt_token_ids, max_model_len)
):
chunked_engine_prompts.append(
token_inputs(prompt_token_ids=chunk_tokens)
chunked_engine_inputs.append(
tokens_input(prompt_token_ids=chunk_tokens)
)
prompt_request_ids.append(
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
)
ctx.engine_prompts = chunked_engine_prompts
ctx.engine_inputs = chunked_engine_inputs
ctx.prompt_request_ids = prompt_request_ids
return None
@@ -184,8 +184,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
if ctx.intermediates is None:
raise ValueError("Original prompts inputs not available")
original_engine_prompts = cast(list[ProcessorInputs], ctx.intermediates)
num_prompts = len(original_engine_prompts)
original_engine_inputs = cast(list[EngineInput], ctx.intermediates)
num_prompts = len(original_engine_inputs)
# Finalize aggregated results
final_res_batch: list[PoolingRequestOutput] = []
@@ -211,12 +211,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = original_engine_prompts[prompt_idx]
original_prompt = original_engine_inputs[prompt_idx]
token_ids = original_prompt.get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInputs."
"not support EmbedsPrompt and EncoderDecoderInput."
)
original_token_ids = cast(list[int], token_ids)
@@ -372,7 +372,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
]
for uri in request.images
]
ctx.engine_prompts = self._batch_render_chat(
ctx.engine_inputs = self._batch_render_chat(
request, all_messages, truncate_prompt_tokens, truncation_side
)
@@ -382,7 +382,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
self._mixed_input_to_messages(inp, task_prefix=task_prefix)
for inp in request.inputs
]
ctx.engine_prompts = self._batch_render_chat(
ctx.engine_inputs = self._batch_render_chat(
request, all_messages, truncate_prompt_tokens, truncation_side
)
@@ -396,7 +396,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
ctx.engine_prompts = self._preprocess_completion_online(
ctx.engine_inputs = self._preprocess_completion_online(
proxy, prompt_input=proxy.input, prompt_embeds=None
)
@@ -406,7 +406,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
all_messages: Sequence[list[ChatCompletionMessageParam]],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> list[ProcessorInputs]:
) -> list[EngineInput]:
"""Batch-render multiple conversations through the chat template."""
if not all_messages:
return []
@@ -438,8 +438,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
_, engine_prompts = renderer.render_chat(all_messages, chat_params, tok_params)
return engine_prompts
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
return engine_inputs
def _validate_input_type(self, input_type: str | None) -> None:
"""Raise if *input_type* is not supported by this model."""
+9 -9
View File
@@ -33,7 +33,7 @@ from vllm.entrypoints.pooling.utils import (
encode_pooling_output_float,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.inputs import ProcessorInputs
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.renderers.inputs.preprocess import prompt_to_seq
@@ -110,7 +110,7 @@ class OpenAIServingPooling(OpenAIServing):
request.task,
)
engine_prompts: Sequence[ProcessorInputs]
engine_inputs: Sequence[EngineInput]
if use_io_processor := isinstance(request, IOProcessorRequest):
if self.io_processor is None:
raise ValueError(
@@ -125,7 +125,7 @@ class OpenAIServingPooling(OpenAIServing):
raw_prompts = await self.io_processor.pre_process_async(
prompt=validated_prompt, request_id=request_id
)
engine_prompts = await self.openai_serving_render.preprocess_cmpl(
engine_inputs = await self.openai_serving_render.preprocess_cmpl(
request,
prompt_to_seq(raw_prompts),
)
@@ -138,7 +138,7 @@ class OpenAIServingPooling(OpenAIServing):
if error_check_ret is not None:
return error_check_ret
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
@@ -146,7 +146,7 @@ class OpenAIServingPooling(OpenAIServing):
default_template_kwargs=None,
)
elif isinstance(request, PoolingCompletionRequest):
engine_prompts = await self.openai_serving_render.preprocess_completion(
engine_inputs = await self.openai_serving_render.preprocess_completion(
request,
prompt_input=request.input,
prompt_embeds=None,
@@ -165,12 +165,12 @@ class OpenAIServingPooling(OpenAIServing):
else:
pooling_params = request.to_pooling_params() # type: ignore
for i, engine_prompt in enumerate(engine_prompts):
for i, engine_input in enumerate(engine_inputs):
request_id_item = f"{request_id}-{i}"
self._log_inputs(
request_id_item,
engine_prompt,
engine_input,
params=pooling_params,
lora_request=lora_request,
)
@@ -182,7 +182,7 @@ class OpenAIServingPooling(OpenAIServing):
)
generator = self.engine_client.encode(
engine_prompt,
engine_input,
pooling_params,
request_id_item,
lora_request=lora_request,
@@ -221,7 +221,7 @@ class OpenAIServingPooling(OpenAIServing):
return IOProcessorResponse(request_id=request_id, data=output)
assert isinstance(request, (PoolingCompletionRequest, PoolingChatRequest))
num_prompts = len(engine_prompts)
num_prompts = len(engine_inputs)
# Non-streaming response
final_res_batch: list[PoolingRequestOutput | None]
+36 -45
View File
@@ -35,7 +35,7 @@ from vllm.entrypoints.pooling.score.utils import (
parse_score_data_single,
validate_score_input,
)
from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs
from vllm.inputs import EngineInput, TokensPrompt, tokens_input
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
@@ -110,12 +110,12 @@ class ServingScores(OpenAIServing):
*(encode_async(t, **tokenization_kwargs) for t in input_texts)
)
engine_prompts: list[ProcessorInputs] = []
engine_inputs: list[EngineInput] = []
for tok_result, input_text in zip(tokenized_prompts, input_texts):
text_token_prompt = self._validate_input(request, tok_result, input_text)
engine_prompts.append(
token_inputs(
engine_inputs.append(
tokens_input(
text_token_prompt["prompt_token_ids"],
prompt=input_text,
)
@@ -125,19 +125,19 @@ class ServingScores(OpenAIServing):
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
pooling_params = request.to_pooling_params("embed")
for i, engine_prompt in enumerate(engine_prompts):
for i, engine_input in enumerate(engine_inputs):
request_id_item = f"{request_id}-{i}"
self._log_inputs(
request_id_item,
engine_prompt,
engine_input,
params=pooling_params,
lora_request=lora_request,
)
generators.append(
self.engine_client.encode(
engine_prompt,
engine_input,
pooling_params,
request_id_item,
lora_request=lora_request,
@@ -151,7 +151,7 @@ class ServingScores(OpenAIServing):
# Non-streaming response
final_res_batch: list[PoolingRequestOutput] = []
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_inputs)
async for i, res in result_generator:
embeddings[i] = res
@@ -183,7 +183,7 @@ class ServingScores(OpenAIServing):
request: RerankRequest | ScoreRequest,
tokenizer: TokenizerLike,
tokenization_kwargs: dict[str, Any],
) -> tuple[str, TokensPrompt]:
) -> TokensPrompt:
"""Parse a single ScoreData into a text + optional multimodal
TokensPrompt for late-interaction encoding.
@@ -197,21 +197,22 @@ class ServingScores(OpenAIServing):
else:
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
prompt_inputs = tokenizer(text, **tokenization_kwargs)
self._validate_input(request, prompt_inputs["input_ids"], text)
prompt_ids = tokenizer.encode(text, **tokenization_kwargs)
self._validate_input(request, prompt_ids, text)
engine_prompt = TokensPrompt(
prompt_token_ids=prompt_inputs["input_ids"],
tok_prompt = TokensPrompt(
prompt_token_ids=prompt_ids,
prompt=text,
)
if mm_data is not None:
engine_prompt["multi_modal_data"] = mm_data
tok_prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
engine_prompt["multi_modal_uuids"] = mm_uuids
tok_prompt["multi_modal_uuids"] = mm_uuids
if request.mm_processor_kwargs is not None:
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
tok_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
return text, engine_prompt
return tok_prompt
async def _late_interaction_score(
self,
@@ -240,7 +241,7 @@ class ServingScores(OpenAIServing):
executor=self._tokenizer_executor,
)
preprocessed = await asyncio.gather(
tok_prompts = await asyncio.gather(
*(
preprocess_async(
data=d,
@@ -253,12 +254,8 @@ class ServingScores(OpenAIServing):
)
)
query_prompts: list[TokensPrompt] = [
prompt for _, prompt in preprocessed[: len(data_1)]
]
doc_prompts: list[TokensPrompt] = [
prompt for _, prompt in preprocessed[len(data_1) :]
]
query_prompts = tok_prompts[: len(data_1)]
doc_prompts = tok_prompts[len(data_1) :]
default_pooling_params = request.to_pooling_params("token_embed")
@@ -268,7 +265,7 @@ class ServingScores(OpenAIServing):
query_prompts
)
query_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
for i, engine_prompt in enumerate(query_prompts):
for i, tok_prompt in enumerate(query_prompts):
request_id_item = f"{request_id}-query-{i}"
pooling_params = default_pooling_params.clone()
pooling_params.late_interaction_params = (
@@ -280,14 +277,14 @@ class ServingScores(OpenAIServing):
self._log_inputs(
request_id_item,
engine_prompt,
tok_prompt,
params=pooling_params,
lora_request=lora_request,
)
query_generators.append(
self.engine_client.encode(
engine_prompt,
tok_prompt,
pooling_params,
request_id_item,
lora_request=lora_request,
@@ -306,7 +303,7 @@ class ServingScores(OpenAIServing):
# stage 2: encode docs and return scalar scores from workers.
doc_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
for i, engine_prompt in enumerate(doc_prompts):
for i, tok_prompt in enumerate(doc_prompts):
request_id_item = f"{request_id}-doc-{i}"
query_idx = 0 if len(query_prompts) == 1 else i
pooling_params = default_pooling_params.clone()
@@ -316,14 +313,14 @@ class ServingScores(OpenAIServing):
self._log_inputs(
request_id_item,
engine_prompt,
tok_prompt,
params=pooling_params,
lora_request=lora_request,
)
doc_generators.append(
self.engine_client.encode(
engine_prompt,
tok_prompt,
pooling_params,
request_id_item,
lora_request=lora_request,
@@ -404,28 +401,22 @@ class ServingScores(OpenAIServing):
)
)
request_prompts: list[str] = []
engine_prompts: list[TokensPrompt] = []
for full_prompt, engine_prompt in preprocessed_prompts:
request_prompts.append(full_prompt)
engine_prompts.append(engine_prompt)
# Schedule the request and get the result generator.
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
default_pooling_params = request.to_pooling_params("classify")
for i, engine_prompt in enumerate(engine_prompts):
for i, (full_prompt, tok_prompt) in enumerate(preprocessed_prompts):
request_id_item = f"{request_id}-{i}"
self._log_inputs(
request_id_item,
request_prompts[i],
full_prompt,
params=default_pooling_params,
lora_request=lora_request,
)
if token_type_ids := engine_prompt.pop("token_type_ids", None):
if token_type_ids := tok_prompt.pop("token_type_ids", None):
pooling_params = default_pooling_params.clone()
compressed = compress_token_type_ids(token_type_ids)
pooling_params.extra_kwargs = {"compressed_token_type_ids": compressed}
@@ -433,7 +424,7 @@ class ServingScores(OpenAIServing):
pooling_params = default_pooling_params
generator = self.engine_client.encode(
engine_prompt,
tok_prompt,
pooling_params,
request_id_item,
lora_request=lora_request,
@@ -447,7 +438,7 @@ class ServingScores(OpenAIServing):
# Non-streaming response
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
engine_prompts
preprocessed_prompts
)
async for i, res in result_generator:
@@ -464,7 +455,7 @@ class ServingScores(OpenAIServing):
data_2: ScoreData,
) -> tuple[str, TokensPrompt]:
model_config = self.model_config
full_prompt, engine_prompt = get_score_prompt(
full_prompt, engine_input = get_score_prompt(
model_config=model_config,
data_1=data_1,
data_2=data_2,
@@ -472,11 +463,11 @@ class ServingScores(OpenAIServing):
tokenization_kwargs=tokenization_kwargs,
score_template=self.score_template,
)
self._validate_input(request, engine_prompt["prompt_token_ids"], full_prompt)
self._validate_input(request, engine_input["prompt_token_ids"], full_prompt)
if request.mm_processor_kwargs is not None:
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
engine_input["mm_processor_kwargs"] = request.mm_processor_kwargs
return full_prompt, engine_prompt
return full_prompt, engine_input
async def _run_scoring(
self,
+7 -3
View File
@@ -20,10 +20,14 @@ from vllm.entrypoints.chat_utils import (
MultiModalItemTracker,
_parse_chat_message_content_parts,
)
from vllm.inputs import TokensPrompt
from vllm.inputs.data import PromptType, TextPrompt
from vllm.inputs import (
MultiModalDataDict,
MultiModalUUIDDict,
PromptType,
TextPrompt,
TokensPrompt,
)
from vllm.model_executor.models.interfaces import supports_score_template
from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.outputs import PoolingRequestOutput
from vllm.renderers.hf import safe_apply_chat_template
from vllm.tokenizers import TokenizerLike
+2 -2
View File
@@ -32,7 +32,7 @@ from vllm.entrypoints.pooling.score.protocol import (
ScoreRequest,
ScoreResponse,
)
from vllm.inputs import ProcessorInputs
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
PoolingCompletionLikeRequest: TypeAlias = (
@@ -74,7 +74,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
engine_prompts: list[ProcessorInputs] | None = None
engine_inputs: list[EngineInput] | None = None
prompt_request_ids: list[str] | None = None
intermediates: Any | None = None