[Frontend][4/n] Improve pooling entrypoints | pooling. (#39153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
+61
-120
@@ -49,9 +49,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
load_chat_template,
|
||||
)
|
||||
from vllm.entrypoints.pooling.io_processor_factories import init_pooling_io_processors
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
ScoringIOProcessor,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import ScoringIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreInput
|
||||
from vllm.entrypoints.pooling.typing import OfflineInputsContext, OfflineOutputsContext
|
||||
from vllm.entrypoints.utils import log_non_default_args
|
||||
@@ -398,12 +396,11 @@ class LLM:
|
||||
self.runner_type = self.model_config.runner_type
|
||||
self.renderer = self.llm_engine.renderer
|
||||
self.chat_template = load_chat_template(chat_template)
|
||||
self.io_processor = self.llm_engine.io_processor
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
|
||||
self.pooling_io_processors = init_pooling_io_processors(
|
||||
supported_tasks=supported_tasks,
|
||||
model_config=self.model_config,
|
||||
vllm_config=self.llm_engine.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
@@ -1081,118 +1078,55 @@ class LLM:
|
||||
pooled hidden states in the same order as the input prompts.
|
||||
"""
|
||||
|
||||
self._verify_pooling_task(pooling_task)
|
||||
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
if self.io_processor is None:
|
||||
raise ValueError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details."
|
||||
)
|
||||
|
||||
# Validate the request data is valid for the loaded plugin
|
||||
prompt_data = prompts.get("data")
|
||||
if prompt_data is None:
|
||||
raise ValueError(
|
||||
"The 'data' field of the prompt is expected to contain "
|
||||
"the prompt data and it cannot be None. "
|
||||
"Refer to the documentation of the IOProcessor "
|
||||
"in use for more details."
|
||||
)
|
||||
validated_prompt = self.io_processor.parse_data(prompt_data)
|
||||
|
||||
# obtain the actual model prompts from the pre-processor
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
|
||||
params_seq: Sequence[PoolingParams] = [
|
||||
self.io_processor.merge_pooling_params(param)
|
||||
for param in self._params_to_seq(
|
||||
pooling_params,
|
||||
len(prompts_seq),
|
||||
)
|
||||
]
|
||||
for p in params_seq:
|
||||
if p.task is None:
|
||||
p.task = "plugin"
|
||||
|
||||
outputs = self._run_completion(
|
||||
prompts=prompts_seq,
|
||||
params=params_seq,
|
||||
output_type=PoolingRequestOutput,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
|
||||
raise ValueError(
|
||||
"The 'data' field is only supported for the 'plugin' pooling task."
|
||||
)
|
||||
self._verify_pooling_task(pooling_task)
|
||||
assert pooling_task is not None and pooling_task in self.pooling_io_processors
|
||||
|
||||
# get the post-processed model outputs
|
||||
assert self.io_processor is not None
|
||||
processed_outputs = self.io_processor.post_process(outputs)
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
|
||||
return [
|
||||
PoolingRequestOutput[Any](
|
||||
request_id="",
|
||||
outputs=processed_outputs,
|
||||
num_cached_tokens=getattr(
|
||||
processed_outputs, "num_cached_tokens", 0
|
||||
),
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
]
|
||||
else:
|
||||
if pooling_params is None:
|
||||
# Use default pooling params.
|
||||
pooling_params = PoolingParams()
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
params_seq = self._params_to_seq(pooling_params, len(prompts_seq))
|
||||
ctx = OfflineInputsContext(
|
||||
prompts=prompts,
|
||||
pooling_params=pooling_params,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif param.task != pooling_task:
|
||||
msg = (
|
||||
f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
engine_inputs = io_processor.pre_process_offline(ctx)
|
||||
n_inputs = len(engine_inputs)
|
||||
assert ctx.pooling_params is not None
|
||||
|
||||
if pooling_task in self.pooling_io_processors:
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
processor_inputs = io_processor.pre_process_offline(
|
||||
ctx=OfflineInputsContext(
|
||||
prompts=prompts_seq, tokenization_kwargs=tokenization_kwargs
|
||||
)
|
||||
)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
lora_request, len(prompts_seq)
|
||||
)
|
||||
seq_priority = self._priority_to_seq(None, len(prompts))
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
|
||||
|
||||
self._render_and_add_requests(
|
||||
prompts=processor_inputs,
|
||||
params=params_seq,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
)
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif pooling_task == "plugin":
|
||||
# `plugin` task uses io_processor.parse_request to verify inputs.
|
||||
# We actually allow plugin to overwrite pooling_task.
|
||||
pass
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
outputs = self._run_engine(
|
||||
use_tqdm=use_tqdm, output_type=PoolingRequestOutput
|
||||
)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs)
|
||||
)
|
||||
else:
|
||||
outputs = self._run_completion(
|
||||
prompts=prompts_seq,
|
||||
params=params_seq,
|
||||
output_type=PoolingRequestOutput,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
|
||||
seq_priority = self._priority_to_seq(None, n_inputs)
|
||||
|
||||
self._render_and_add_requests(
|
||||
prompts=engine_inputs,
|
||||
params=params_seq,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
)
|
||||
|
||||
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def _verify_pooling_task(self, pooling_task: PoolingTask | None):
|
||||
@@ -1254,6 +1188,14 @@ class LLM:
|
||||
pooling_task,
|
||||
)
|
||||
|
||||
if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
|
||||
raise ValueError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details."
|
||||
)
|
||||
|
||||
def embed(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
@@ -1458,6 +1400,9 @@ class LLM:
|
||||
scoring_data = io_processor.valid_inputs(data_1, data_2)
|
||||
n_queries = len(scoring_data.data_1)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
ctx = OfflineInputsContext(
|
||||
prompts=scoring_data,
|
||||
pooling_params=pooling_params,
|
||||
@@ -1466,15 +1411,11 @@ class LLM:
|
||||
n_queries=n_queries,
|
||||
)
|
||||
|
||||
processor_inputs = io_processor.pre_process_offline(ctx)
|
||||
engine_inputs = io_processor.pre_process_offline(ctx)
|
||||
n_inputs = len(engine_inputs)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
lora_request, len(processor_inputs)
|
||||
)
|
||||
|
||||
if ctx.pooling_params is None:
|
||||
ctx.pooling_params = PoolingParams()
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, len(processor_inputs))
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
@@ -1483,10 +1424,10 @@ class LLM:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
seq_priority = self._priority_to_seq(None, len(processor_inputs))
|
||||
seq_priority = self._priority_to_seq(None, n_inputs)
|
||||
|
||||
self._render_and_add_requests(
|
||||
prompts=processor_inputs,
|
||||
prompts=engine_inputs,
|
||||
params=params_seq,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
@@ -1579,7 +1520,7 @@ class LLM:
|
||||
if isinstance(params, Sequence):
|
||||
if len(params) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({params}) "
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and params ({len(params)}) must be the same."
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user