[Frontend][4/n] Improve pooling entrypoints | pooling. (#39153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -4,8 +4,8 @@
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from vllm import PoolingRequestOutput, PromptType
|
||||
from vllm.config import ModelConfig
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateConfig,
|
||||
@@ -33,11 +33,12 @@ class PoolingIOProcessor:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
):
|
||||
self.model_config = model_config
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.renderer = renderer
|
||||
|
||||
self.chat_template = chat_template_config.chat_template
|
||||
@@ -48,12 +49,12 @@ class PoolingIOProcessor:
|
||||
chat_template_config.trust_request_chat_template
|
||||
)
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params()
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params()
|
||||
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
request = ctx.request
|
||||
|
||||
@@ -100,12 +101,16 @@ class PoolingIOProcessor:
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
assert not isinstance(ctx.prompts, ScoringData)
|
||||
assert not isinstance(ctx.prompts, ScoringData) and not (
|
||||
isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
)
|
||||
|
||||
prompts_seq = prompt_to_seq(ctx.prompts)
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
return self._preprocess_completion_offline(
|
||||
prompts=ctx.prompts, tok_params=tok_params
|
||||
prompts=prompts_seq, tok_params=tok_params
|
||||
)
|
||||
|
||||
async def pre_process_offline_async(self, ctx: OfflineInputsContext):
|
||||
@@ -243,3 +248,19 @@ class PoolingIOProcessor:
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
return None
|
||||
|
||||
def _params_to_seq(
|
||||
self,
|
||||
params: PoolingParams | Sequence[PoolingParams],
|
||||
num_requests: int,
|
||||
) -> Sequence[PoolingParams]:
|
||||
if isinstance(params, Sequence):
|
||||
if len(params) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and params ({len(params)}) must be the same."
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
return [params] * num_requests
|
||||
|
||||
Reference in New Issue
Block a user