[Frontend][4/n] Improve pooling entrypoints | pooling. (#39153)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-04-09 10:09:45 +00:00
committed by GitHub
parent b6c9be509e
commit 66c079ae83
43 changed files with 554 additions and 733 deletions
+30 -9
View File
@@ -4,8 +4,8 @@
from collections.abc import Sequence
from typing import Any, Final
from vllm import PoolingRequestOutput, PromptType
from vllm.config import ModelConfig
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateConfig,
@@ -33,11 +33,12 @@ class PoolingIOProcessor:
def __init__(
self,
model_config: ModelConfig,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
):
self.model_config = model_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.renderer = renderer
self.chat_template = chat_template_config.chat_template
@@ -48,12 +49,12 @@ class PoolingIOProcessor:
chat_template_config.trust_request_chat_template
)
def create_pooling_params(self, request):
return request.to_pooling_params()
#######################################
# online APIs
def create_pooling_params(self, request):
return request.to_pooling_params()
def pre_process_online(self, ctx: PoolingServeContext):
request = ctx.request
@@ -100,12 +101,16 @@ class PoolingIOProcessor:
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert not isinstance(ctx.prompts, ScoringData)
assert not isinstance(ctx.prompts, ScoringData) and not (
isinstance(ctx.prompts, dict) and "data" in ctx.prompts
)
prompts_seq = prompt_to_seq(ctx.prompts)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
return self._preprocess_completion_offline(
prompts=ctx.prompts, tok_params=tok_params
prompts=prompts_seq, tok_params=tok_params
)
async def pre_process_offline_async(self, ctx: OfflineInputsContext):
@@ -243,3 +248,19 @@ class PoolingIOProcessor:
"Refused request with untrusted chat template."
)
return None
def _params_to_seq(
self,
params: PoolingParams | Sequence[PoolingParams],
num_requests: int,
) -> Sequence[PoolingParams]:
if isinstance(params, Sequence):
if len(params) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and params ({len(params)}) must be the same."
)
return params
return [params] * num_requests
+47 -26
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Mapping
from http import HTTPStatus
from typing import ClassVar
@@ -9,7 +11,7 @@ from fastapi.responses import Response
from starlette.datastructures import Headers
from vllm import PoolingParams, PoolingRequestOutput, envs
from vllm.config import ModelConfig
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatTemplateConfig,
@@ -35,7 +37,7 @@ from vllm.utils.async_utils import merge_async_iterators
from .io_processor import PoolingIOProcessor
class PoolingServing:
class PoolingServingBase(ABC):
request_id_prefix: ClassVar[str]
def __init__(
@@ -50,10 +52,11 @@ class PoolingServing:
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
super().__init__()
self.engine_client = engine_client
self.models = models
self.model_config = models.model_config
self.renderer = models.renderer
self.vllm_config = engine_client.vllm_config
self.max_model_len = self.model_config.max_model_len
self.request_logger = request_logger
self.return_tokens_as_token_ids = return_tokens_as_token_ids
@@ -63,31 +66,14 @@ class PoolingServing:
chat_template_content_format=chat_template_content_format,
trust_request_chat_template=trust_request_chat_template,
)
self.io_processor = self.init_io_processor(
model_config=models.model_config,
renderer=models.renderer,
chat_template_config=self.chat_template_config,
)
def init_io_processor(
self,
model_config: ModelConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> PoolingIOProcessor:
raise NotImplementedError
@abstractmethod
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
ctx = await self._init_ctx(request, raw_request)
await self.io_processor.pre_process_online_async(ctx)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
await self.io_processor.post_process_online_async(ctx)
return await self._build_response(ctx)
raise NotImplementedError
async def _init_ctx(
self,
@@ -124,10 +110,8 @@ class PoolingServing:
else await self._get_trace_headers(ctx.raw_request.headers)
)
if ctx.pooling_params is None:
pooling_params = self.io_processor.create_pooling_params(ctx.request)
else:
pooling_params = ctx.pooling_params
assert ctx.pooling_params is not None
pooling_params = ctx.pooling_params
if isinstance(pooling_params, list):
for params in pooling_params:
@@ -190,6 +174,7 @@ class PoolingServing:
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
@abstractmethod
async def _build_response(
self,
ctx: PoolingServeContext,
@@ -355,3 +340,39 @@ class PoolingServing:
params=params,
lora_request=lora_request,
)
class PoolingServing(PoolingServingBase, ABC):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.io_processor = self.init_io_processor(
vllm_config=self.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
@abstractmethod
def init_io_processor(
self,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> PoolingIOProcessor:
raise NotImplementedError
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
ctx = await self._init_ctx(request, raw_request)
await self.io_processor.pre_process_online_async(ctx)
if ctx.pooling_params is None:
ctx.pooling_params = self.io_processor.create_pooling_params(request)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
await self.io_processor.post_process_online_async(ctx)
return await self._build_response(ctx)