[Model][2/N] Improve all pooling task | Support multi-vector retrieval (#25370)
Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
+18
-19
@@ -951,7 +951,7 @@ class LLM:
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
pooling_task: PoolingTask = "encode",
|
||||
pooling_task: PoolingTask | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""Apply pooling to the hidden states corresponding to the input
|
||||
@@ -986,25 +986,24 @@ class LLM:
|
||||
instead pass them via the `inputs` parameter.
|
||||
"""
|
||||
|
||||
if self.supported_tasks == ["encode"] and pooling_task is None:
|
||||
pooling_task = "encode"
|
||||
error_str = (
|
||||
"pooling_task required for `LLM.encode`\n"
|
||||
"Please use one of the more specific methods or set the "
|
||||
"pooling_task when using `LLM.encode`:\n"
|
||||
" - For embeddings, use `LLM.embed(...)` "
|
||||
'or `pooling_task="embed"`.\n'
|
||||
" - For classification logits, use `LLM.classify(...)` "
|
||||
'or `pooling_task="classify"`.\n'
|
||||
" - For similarity scores, use `LLM.score(...)`.\n"
|
||||
" - For rewards, use `LLM.reward(...)` "
|
||||
'or `pooling_task="token_classify"`\n'
|
||||
" - For token classification, "
|
||||
'use `pooling_task="token_classify"`\n'
|
||||
' - For multi-vector retrieval, use `pooling_task="token_embed"`'
|
||||
)
|
||||
|
||||
if pooling_task is None:
|
||||
pooling_task = "embed" if "embed" in self.supported_tasks else "encode"
|
||||
|
||||
logger.warning_once(
|
||||
"`LLM.encode` is currently using `pooling_task = %s`.\n"
|
||||
"Please use one of the more specific methods or set the "
|
||||
"task directly when using `LLM.encode`:\n"
|
||||
" - For embeddings, use `LLM.embed(...)` "
|
||||
'or `pooling_task="embed"`.\n'
|
||||
" - For classification logits, use `LLM.classify(...)` "
|
||||
'or `pooling_task="classify"`.\n'
|
||||
" - For rewards, use `LLM.reward(...)` "
|
||||
'or `pooling_task="reward"`\n'
|
||||
" - For similarity scores, use `LLM.score(...)`.",
|
||||
pooling_task,
|
||||
)
|
||||
raise ValueError(error_str)
|
||||
|
||||
model_config = self.model_config
|
||||
runner_type = model_config.runner_type
|
||||
@@ -1206,7 +1205,7 @@ class LLM:
|
||||
lora_request=lora_request,
|
||||
pooling_params=pooling_params,
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
pooling_task="encode",
|
||||
pooling_task="token_classify",
|
||||
)
|
||||
|
||||
def _embedding_score(
|
||||
|
||||
@@ -1748,16 +1748,19 @@ async def init_app_state(
|
||||
else None
|
||||
)
|
||||
state.openai_serving_pooling = (
|
||||
OpenAIServingPooling(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
log_error_stack=args.log_error_stack,
|
||||
(
|
||||
OpenAIServingPooling(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
supported_tasks=supported_tasks,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
)
|
||||
if "encode" in supported_tasks
|
||||
if ("token_embed" in supported_tasks or "token_classify" in supported_tasks)
|
||||
else None
|
||||
)
|
||||
state.openai_serving_embedding = (
|
||||
|
||||
@@ -1682,7 +1682,7 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
|
||||
When using plugins IOProcessor plugins, the actual input is processed
|
||||
by the plugin itself. Hence, we use a generic type for the request data
|
||||
"""
|
||||
softmax: bool = True
|
||||
activation: bool = False
|
||||
|
||||
embed_dtype: str = Field(
|
||||
default="float32",
|
||||
@@ -1693,7 +1693,7 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(task="encode", softmax=self.softmax)
|
||||
return PoolingParams(task="token_classify", activation=self.activation)
|
||||
|
||||
|
||||
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.entrypoints.renderer import RenderConfig
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils import merge_async_iterators
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -62,6 +63,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
@@ -75,6 +77,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
log_error_stack=log_error_stack,
|
||||
)
|
||||
|
||||
self.supported_tasks = supported_tasks
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
@@ -178,8 +181,17 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
try:
|
||||
pooling_params = request.to_pooling_params()
|
||||
|
||||
if "token_embed" in self.supported_tasks:
|
||||
pooling_task = "token_embed"
|
||||
elif "token_classify" in self.supported_tasks:
|
||||
pooling_task = "token_classify"
|
||||
else:
|
||||
return self.create_error_response(
|
||||
f"pooling_task must be one of {self.supported_tasks}."
|
||||
)
|
||||
|
||||
try:
|
||||
pooling_params.verify("encode", self.model_config)
|
||||
pooling_params.verify(pooling_task, self.model_config)
|
||||
except ValueError as e:
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user