[Frontend][4/N] Improve all pooling task | Add plugin pooling task (#26973)
Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: Christian Pinto <christian.pinto@ibm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Christian Pinto <christian.pinto@ibm.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Christian Pinto
parent
fe2016de2d
commit
3fa2c12185
+28
-13
@@ -1024,19 +1024,6 @@ class LLM:
|
||||
"pooling model."
|
||||
)
|
||||
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise ValueError(f"pooling_task must be one of {self.supported_tasks}.")
|
||||
|
||||
if pooling_params is None:
|
||||
# Use default pooling params.
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
for param in as_iter(pooling_params):
|
||||
param.verify(pooling_task, model_config)
|
||||
# for backwards compatibility
|
||||
if truncate_prompt_tokens is not None:
|
||||
param.truncate_prompt_tokens = truncate_prompt_tokens
|
||||
|
||||
io_processor_prompt = False
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
io_processor_prompt = True
|
||||
@@ -1054,6 +1041,34 @@ class LLM:
|
||||
# obtain the actual model prompts from the pre-processor
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
|
||||
if io_processor_prompt:
|
||||
assert self.io_processor is not None
|
||||
if is_list_of(pooling_params, PoolingParams):
|
||||
validated_pooling_params: list[PoolingParams] = []
|
||||
for param in as_iter(pooling_params):
|
||||
validated_pooling_params.append(
|
||||
self.io_processor.validate_or_generate_params(param)
|
||||
)
|
||||
pooling_params = validated_pooling_params
|
||||
else:
|
||||
assert not isinstance(pooling_params, Sequence)
|
||||
pooling_params = self.io_processor.validate_or_generate_params(
|
||||
pooling_params
|
||||
)
|
||||
else:
|
||||
if pooling_params is None:
|
||||
# Use default pooling params.
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise ValueError(f"pooling_task must be one of {self.supported_tasks}.")
|
||||
|
||||
for param in as_iter(pooling_params):
|
||||
param.verify(pooling_task, model_config)
|
||||
# for backwards compatibility
|
||||
if truncate_prompt_tokens is not None:
|
||||
param.truncate_prompt_tokens = truncate_prompt_tokens
|
||||
|
||||
self._validate_and_add_requests(
|
||||
prompts=prompts,
|
||||
params=pooling_params,
|
||||
|
||||
@@ -1748,7 +1748,12 @@ async def init_app_state(
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
)
|
||||
if ("token_embed" in supported_tasks or "token_classify" in supported_tasks)
|
||||
if (
|
||||
any(
|
||||
task in supported_tasks
|
||||
for task in ["token_embed", "token_classify", "plugin"]
|
||||
)
|
||||
)
|
||||
else None
|
||||
)
|
||||
state.openai_serving_embedding = (
|
||||
|
||||
@@ -1707,11 +1707,6 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
|
||||
if the served model does not use priority scheduling.
|
||||
"""
|
||||
data: T
|
||||
"""
|
||||
When using plugins IOProcessor plugins, the actual input is processed
|
||||
by the plugin itself. Hence, we use a generic type for the request data
|
||||
"""
|
||||
activation: bool = False
|
||||
|
||||
encoding_format: EncodingFormat = "float"
|
||||
embed_dtype: EmbedDType = Field(
|
||||
@@ -1732,7 +1727,7 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(task="token_classify", activation=self.activation)
|
||||
return PoolingParams()
|
||||
|
||||
|
||||
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.entrypoints.renderer import RenderConfig
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tasks import PoolingTask, SupportedTask
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.serial_utils import (
|
||||
EmbedDType,
|
||||
@@ -161,12 +161,21 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
# Schedule the request and get the result generator.
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
try:
|
||||
pooling_params = request.to_pooling_params()
|
||||
if is_io_processor_request:
|
||||
assert self.io_processor is not None and isinstance(
|
||||
request, IOProcessorRequest
|
||||
)
|
||||
pooling_params = self.io_processor.validate_or_generate_params()
|
||||
else:
|
||||
pooling_params = request.to_pooling_params()
|
||||
|
||||
pooling_task: PoolingTask
|
||||
if "token_embed" in self.supported_tasks:
|
||||
pooling_task = "token_embed"
|
||||
elif "token_classify" in self.supported_tasks:
|
||||
pooling_task = "token_classify"
|
||||
elif "plugin" in self.supported_tasks:
|
||||
pooling_task = "plugin"
|
||||
else:
|
||||
return self.create_error_response(
|
||||
f"pooling_task must be one of {self.supported_tasks}."
|
||||
|
||||
Reference in New Issue
Block a user