[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -18,7 +18,7 @@ from vllm.entrypoints.pooling.typing import (
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.inputs.data import ProcessorInputs, SingletonPrompt
|
||||
from vllm.inputs import EngineInput, SingletonPrompt
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.tool_parsers import ToolParser
|
||||
@@ -60,7 +60,7 @@ class PoolingIOProcessor:
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
_, engine_prompts = self._preprocess_chat_online(
|
||||
_, engine_inputs = self._preprocess_chat_online(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
@@ -68,7 +68,7 @@ class PoolingIOProcessor:
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||
engine_prompts = self._preprocess_completion_online(
|
||||
engine_inputs = self._preprocess_completion_online(
|
||||
request,
|
||||
prompt_input=request.input,
|
||||
prompt_embeds=None,
|
||||
@@ -76,7 +76,7 @@ class PoolingIOProcessor:
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
ctx.engine_prompts = engine_prompts
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
async def pre_process_online_async(self, ctx: PoolingServeContext):
|
||||
self.pre_process_online(ctx)
|
||||
@@ -100,7 +100,7 @@ class PoolingIOProcessor:
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[ProcessorInputs]:
|
||||
) -> Sequence[EngineInput]:
|
||||
return self._preprocess_completion_offline(
|
||||
prompts=prompts, tokenization_kwargs=tokenization_kwargs
|
||||
)
|
||||
@@ -128,7 +128,7 @@ class PoolingIOProcessor:
|
||||
request: RendererRequest,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
) -> list[ProcessorInputs]:
|
||||
) -> list[EngineInput]:
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -167,7 +167,7 @@ class PoolingIOProcessor:
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: type[ToolParser] | None = None,
|
||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
renderer = self.renderer
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
@@ -188,7 +188,7 @@ class PoolingIOProcessor:
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_prompt,) = renderer.render_chat(
|
||||
(conversation,), (engine_input,) = renderer.render_chat(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
@@ -199,13 +199,13 @@ class PoolingIOProcessor:
|
||||
},
|
||||
)
|
||||
|
||||
return conversation, [engine_prompt]
|
||||
return conversation, [engine_input]
|
||||
|
||||
def _preprocess_completion_offline(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[ProcessorInputs]:
|
||||
) -> Sequence[EngineInput]:
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest, PoolingServeContext
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.inputs.data import ProcessorInputs
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
@@ -106,7 +106,7 @@ class PoolingServing:
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_prompts is None:
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
@@ -120,7 +120,7 @@ class PoolingServing:
|
||||
pooling_params = self.io_processor.create_pooling_params(ctx.request)
|
||||
pooling_params.verify(self.model_config)
|
||||
|
||||
for i, engine_prompt in enumerate(ctx.engine_prompts):
|
||||
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||
prompt_request_id = (
|
||||
f"{ctx.request_id}-{i}"
|
||||
if ctx.prompt_request_ids is None
|
||||
@@ -129,13 +129,13 @@ class PoolingServing:
|
||||
|
||||
self._log_inputs(
|
||||
prompt_request_id,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
pooling_params,
|
||||
prompt_request_id,
|
||||
lora_request=ctx.lora_request,
|
||||
@@ -151,13 +151,13 @@ class PoolingServing:
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_prompts is None:
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
if ctx.result_generator is None:
|
||||
raise ValueError("Result generator not available")
|
||||
|
||||
num_inputs = len(ctx.engine_prompts)
|
||||
num_inputs = len(ctx.engine_inputs)
|
||||
final_res_batch: list[PoolingRequestOutput | None]
|
||||
final_res_batch = [None] * num_inputs
|
||||
|
||||
@@ -317,7 +317,7 @@ class PoolingServing:
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: ProcessorInputs,
|
||||
inputs: EngineInput,
|
||||
params: PoolingParams,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.inputs.data import ProcessorInputs, token_inputs
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.renderers import merge_kwargs
|
||||
@@ -83,20 +83,20 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
#################################################################
|
||||
|
||||
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
if ctx.engine_prompts is None:
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
ctx.intermediates = ctx.engine_prompts
|
||||
ctx.intermediates = ctx.engine_inputs
|
||||
request_id = ctx.request_id
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_prompts: list[ProcessorInputs] = []
|
||||
chunked_engine_inputs: list[EngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
for prompt_idx, engine_prompt in enumerate(ctx.engine_prompts):
|
||||
token_ids = engine_prompt.get("prompt_token_ids", None)
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input.get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInputs."
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
prompt_token_ids = cast(list[int], token_ids)
|
||||
@@ -104,14 +104,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
for chunk_idx, chunk_tokens in enumerate(
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_prompts.append(
|
||||
token_inputs(prompt_token_ids=chunk_tokens)
|
||||
chunked_engine_inputs.append(
|
||||
tokens_input(prompt_token_ids=chunk_tokens)
|
||||
)
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
)
|
||||
|
||||
ctx.engine_prompts = chunked_engine_prompts
|
||||
ctx.engine_inputs = chunked_engine_inputs
|
||||
ctx.prompt_request_ids = prompt_request_ids
|
||||
|
||||
return None
|
||||
@@ -184,8 +184,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
if ctx.intermediates is None:
|
||||
raise ValueError("Original prompts inputs not available")
|
||||
|
||||
original_engine_prompts = cast(list[ProcessorInputs], ctx.intermediates)
|
||||
num_prompts = len(original_engine_prompts)
|
||||
original_engine_inputs = cast(list[EngineInput], ctx.intermediates)
|
||||
num_prompts = len(original_engine_inputs)
|
||||
|
||||
# Finalize aggregated results
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
@@ -211,12 +211,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
pooling_output_data = PoolingOutput(data=final_embedding)
|
||||
|
||||
# Get original prompt token IDs for this prompt
|
||||
original_prompt = original_engine_prompts[prompt_idx]
|
||||
original_prompt = original_engine_inputs[prompt_idx]
|
||||
token_ids = original_prompt.get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInputs."
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
original_token_ids = cast(list[int], token_ids)
|
||||
@@ -372,7 +372,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
]
|
||||
for uri in request.images
|
||||
]
|
||||
ctx.engine_prompts = self._batch_render_chat(
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||
)
|
||||
|
||||
@@ -382,7 +382,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix)
|
||||
for inp in request.inputs
|
||||
]
|
||||
ctx.engine_prompts = self._batch_render_chat(
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||
)
|
||||
|
||||
@@ -396,7 +396,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
ctx.engine_prompts = self._preprocess_completion_online(
|
||||
ctx.engine_inputs = self._preprocess_completion_online(
|
||||
proxy, prompt_input=proxy.input, prompt_embeds=None
|
||||
)
|
||||
|
||||
@@ -406,7 +406,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> list[ProcessorInputs]:
|
||||
) -> list[EngineInput]:
|
||||
"""Batch-render multiple conversations through the chat template."""
|
||||
if not all_messages:
|
||||
return []
|
||||
@@ -438,8 +438,8 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
_, engine_prompts = renderer.render_chat(all_messages, chat_params, tok_params)
|
||||
return engine_prompts
|
||||
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
|
||||
return engine_inputs
|
||||
|
||||
def _validate_input_type(self, input_type: str | None) -> None:
|
||||
"""Raise if *input_type* is not supported by this model."""
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.entrypoints.pooling.utils import (
|
||||
encode_pooling_output_float,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.inputs import ProcessorInputs
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.renderers.inputs.preprocess import prompt_to_seq
|
||||
@@ -110,7 +110,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
request.task,
|
||||
)
|
||||
|
||||
engine_prompts: Sequence[ProcessorInputs]
|
||||
engine_inputs: Sequence[EngineInput]
|
||||
if use_io_processor := isinstance(request, IOProcessorRequest):
|
||||
if self.io_processor is None:
|
||||
raise ValueError(
|
||||
@@ -125,7 +125,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
raw_prompts = await self.io_processor.pre_process_async(
|
||||
prompt=validated_prompt, request_id=request_id
|
||||
)
|
||||
engine_prompts = await self.openai_serving_render.preprocess_cmpl(
|
||||
engine_inputs = await self.openai_serving_render.preprocess_cmpl(
|
||||
request,
|
||||
prompt_to_seq(raw_prompts),
|
||||
)
|
||||
@@ -138,7 +138,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
_, engine_prompts = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
@@ -146,7 +146,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
elif isinstance(request, PoolingCompletionRequest):
|
||||
engine_prompts = await self.openai_serving_render.preprocess_completion(
|
||||
engine_inputs = await self.openai_serving_render.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.input,
|
||||
prompt_embeds=None,
|
||||
@@ -165,12 +165,12 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
else:
|
||||
pooling_params = request.to_pooling_params() # type: ignore
|
||||
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
request_id_item = f"{request_id}-{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
@@ -182,7 +182,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
)
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -221,7 +221,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
return IOProcessorResponse(request_id=request_id, data=output)
|
||||
|
||||
assert isinstance(request, (PoolingCompletionRequest, PoolingChatRequest))
|
||||
num_prompts = len(engine_prompts)
|
||||
num_prompts = len(engine_inputs)
|
||||
|
||||
# Non-streaming response
|
||||
final_res_batch: list[PoolingRequestOutput | None]
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.entrypoints.pooling.score.utils import (
|
||||
parse_score_data_single,
|
||||
validate_score_input,
|
||||
)
|
||||
from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs
|
||||
from vllm.inputs import EngineInput, TokensPrompt, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||
@@ -110,12 +110,12 @@ class ServingScores(OpenAIServing):
|
||||
*(encode_async(t, **tokenization_kwargs) for t in input_texts)
|
||||
)
|
||||
|
||||
engine_prompts: list[ProcessorInputs] = []
|
||||
engine_inputs: list[EngineInput] = []
|
||||
for tok_result, input_text in zip(tokenized_prompts, input_texts):
|
||||
text_token_prompt = self._validate_input(request, tok_result, input_text)
|
||||
|
||||
engine_prompts.append(
|
||||
token_inputs(
|
||||
engine_inputs.append(
|
||||
tokens_input(
|
||||
text_token_prompt["prompt_token_ids"],
|
||||
prompt=input_text,
|
||||
)
|
||||
@@ -125,19 +125,19 @@ class ServingScores(OpenAIServing):
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
pooling_params = request.to_pooling_params("embed")
|
||||
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
request_id_item = f"{request_id}-{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
generators.append(
|
||||
self.engine_client.encode(
|
||||
engine_prompt,
|
||||
engine_input,
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -151,7 +151,7 @@ class ServingScores(OpenAIServing):
|
||||
# Non-streaming response
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
|
||||
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_inputs)
|
||||
|
||||
async for i, res in result_generator:
|
||||
embeddings[i] = res
|
||||
@@ -183,7 +183,7 @@ class ServingScores(OpenAIServing):
|
||||
request: RerankRequest | ScoreRequest,
|
||||
tokenizer: TokenizerLike,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
) -> TokensPrompt:
|
||||
"""Parse a single ScoreData into a text + optional multimodal
|
||||
TokensPrompt for late-interaction encoding.
|
||||
|
||||
@@ -197,21 +197,22 @@ class ServingScores(OpenAIServing):
|
||||
else:
|
||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||
|
||||
prompt_inputs = tokenizer(text, **tokenization_kwargs)
|
||||
self._validate_input(request, prompt_inputs["input_ids"], text)
|
||||
prompt_ids = tokenizer.encode(text, **tokenization_kwargs)
|
||||
self._validate_input(request, prompt_ids, text)
|
||||
|
||||
engine_prompt = TokensPrompt(
|
||||
prompt_token_ids=prompt_inputs["input_ids"],
|
||||
tok_prompt = TokensPrompt(
|
||||
prompt_token_ids=prompt_ids,
|
||||
prompt=text,
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
engine_prompt["multi_modal_data"] = mm_data
|
||||
tok_prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
engine_prompt["multi_modal_uuids"] = mm_uuids
|
||||
tok_prompt["multi_modal_uuids"] = mm_uuids
|
||||
if request.mm_processor_kwargs is not None:
|
||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
tok_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
|
||||
return text, engine_prompt
|
||||
return tok_prompt
|
||||
|
||||
async def _late_interaction_score(
|
||||
self,
|
||||
@@ -240,7 +241,7 @@ class ServingScores(OpenAIServing):
|
||||
executor=self._tokenizer_executor,
|
||||
)
|
||||
|
||||
preprocessed = await asyncio.gather(
|
||||
tok_prompts = await asyncio.gather(
|
||||
*(
|
||||
preprocess_async(
|
||||
data=d,
|
||||
@@ -253,12 +254,8 @@ class ServingScores(OpenAIServing):
|
||||
)
|
||||
)
|
||||
|
||||
query_prompts: list[TokensPrompt] = [
|
||||
prompt for _, prompt in preprocessed[: len(data_1)]
|
||||
]
|
||||
doc_prompts: list[TokensPrompt] = [
|
||||
prompt for _, prompt in preprocessed[len(data_1) :]
|
||||
]
|
||||
query_prompts = tok_prompts[: len(data_1)]
|
||||
doc_prompts = tok_prompts[len(data_1) :]
|
||||
|
||||
default_pooling_params = request.to_pooling_params("token_embed")
|
||||
|
||||
@@ -268,7 +265,7 @@ class ServingScores(OpenAIServing):
|
||||
query_prompts
|
||||
)
|
||||
query_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
for i, engine_prompt in enumerate(query_prompts):
|
||||
for i, tok_prompt in enumerate(query_prompts):
|
||||
request_id_item = f"{request_id}-query-{i}"
|
||||
pooling_params = default_pooling_params.clone()
|
||||
pooling_params.late_interaction_params = (
|
||||
@@ -280,14 +277,14 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_prompt,
|
||||
tok_prompt,
|
||||
params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
query_generators.append(
|
||||
self.engine_client.encode(
|
||||
engine_prompt,
|
||||
tok_prompt,
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -306,7 +303,7 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
doc_generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
for i, engine_prompt in enumerate(doc_prompts):
|
||||
for i, tok_prompt in enumerate(doc_prompts):
|
||||
request_id_item = f"{request_id}-doc-{i}"
|
||||
query_idx = 0 if len(query_prompts) == 1 else i
|
||||
pooling_params = default_pooling_params.clone()
|
||||
@@ -316,14 +313,14 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
engine_prompt,
|
||||
tok_prompt,
|
||||
params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
doc_generators.append(
|
||||
self.engine_client.encode(
|
||||
engine_prompt,
|
||||
tok_prompt,
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -404,28 +401,22 @@ class ServingScores(OpenAIServing):
|
||||
)
|
||||
)
|
||||
|
||||
request_prompts: list[str] = []
|
||||
engine_prompts: list[TokensPrompt] = []
|
||||
for full_prompt, engine_prompt in preprocessed_prompts:
|
||||
request_prompts.append(full_prompt)
|
||||
engine_prompts.append(engine_prompt)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
default_pooling_params = request.to_pooling_params("classify")
|
||||
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
for i, (full_prompt, tok_prompt) in enumerate(preprocessed_prompts):
|
||||
request_id_item = f"{request_id}-{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
request_prompts[i],
|
||||
full_prompt,
|
||||
params=default_pooling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
if token_type_ids := tok_prompt.pop("token_type_ids", None):
|
||||
pooling_params = default_pooling_params.clone()
|
||||
compressed = compress_token_type_ids(token_type_ids)
|
||||
pooling_params.extra_kwargs = {"compressed_token_type_ids": compressed}
|
||||
@@ -433,7 +424,7 @@ class ServingScores(OpenAIServing):
|
||||
pooling_params = default_pooling_params
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
engine_prompt,
|
||||
tok_prompt,
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_request,
|
||||
@@ -447,7 +438,7 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
# Non-streaming response
|
||||
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
|
||||
engine_prompts
|
||||
preprocessed_prompts
|
||||
)
|
||||
|
||||
async for i, res in result_generator:
|
||||
@@ -464,7 +455,7 @@ class ServingScores(OpenAIServing):
|
||||
data_2: ScoreData,
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
model_config = self.model_config
|
||||
full_prompt, engine_prompt = get_score_prompt(
|
||||
full_prompt, engine_input = get_score_prompt(
|
||||
model_config=model_config,
|
||||
data_1=data_1,
|
||||
data_2=data_2,
|
||||
@@ -472,11 +463,11 @@ class ServingScores(OpenAIServing):
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
score_template=self.score_template,
|
||||
)
|
||||
self._validate_input(request, engine_prompt["prompt_token_ids"], full_prompt)
|
||||
self._validate_input(request, engine_input["prompt_token_ids"], full_prompt)
|
||||
if request.mm_processor_kwargs is not None:
|
||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
engine_input["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
|
||||
return full_prompt, engine_prompt
|
||||
return full_prompt, engine_input
|
||||
|
||||
async def _run_scoring(
|
||||
self,
|
||||
|
||||
@@ -20,10 +20,14 @@ from vllm.entrypoints.chat_utils import (
|
||||
MultiModalItemTracker,
|
||||
_parse_chat_message_content_parts,
|
||||
)
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.inputs.data import PromptType, TextPrompt
|
||||
from vllm.inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalUUIDDict,
|
||||
PromptType,
|
||||
TextPrompt,
|
||||
TokensPrompt,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.entrypoints.pooling.score.protocol import (
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.inputs import ProcessorInputs
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
PoolingCompletionLikeRequest: TypeAlias = (
|
||||
@@ -74,7 +74,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
|
||||
engine_prompts: list[ProcessorInputs] | None = None
|
||||
engine_inputs: list[EngineInput] | None = None
|
||||
prompt_request_ids: list[str] | None = None
|
||||
intermediates: Any | None = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user