[Core] Set pooling params based on task and model (#21128)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-07-18 05:41:17 -07:00
committed by GitHub
parent 4adc66f64d
commit 45badd05d0
24 changed files with 509 additions and 241 deletions
+15 -3
View File
@@ -24,6 +24,7 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.logger import init_logger
from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
PoolingRequestOutput)
from vllm.pooling_params import PoolingParams
logger = init_logger(__name__)
@@ -45,6 +46,7 @@ def _get_embedding(
class EmbeddingMixin(OpenAIServing):
@override
async def _preprocess(
self,
ctx: ServeContext,
@@ -97,6 +99,7 @@ class EmbeddingMixin(OpenAIServing):
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
@override
def _build_response(
self,
ctx: ServeContext,
@@ -191,11 +194,20 @@ class OpenAIServingEmbedding(EmbeddingMixin):
ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens
pooling_params = ctx.request.to_pooling_params()
return None
@override
def _create_pooling_params(
self,
ctx: ServeContext[EmbeddingRequest],
) -> Union[PoolingParams, ErrorResponse]:
pooling_params = super()._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
try:
pooling_params.verify(self.model_config)
pooling_params.verify("embed", self.model_config)
except ValueError as e:
return self.create_error_response(str(e))
return None
return pooling_params