[Frontend][4/n] Improve pooling entrypoints | pooling. (#39153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -8,8 +8,6 @@ from typing import Literal, TypeAlias, cast
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServing
|
||||
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
|
||||
@@ -33,12 +31,10 @@ from vllm.entrypoints.pooling.utils import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
JSONResponseCLS = get_json_response_cls()
|
||||
|
||||
EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
|
||||
|
||||
@@ -49,17 +45,13 @@ class ServingEmbedding(PoolingServing):
|
||||
request_id_prefix = "embd"
|
||||
io_processor: EmbedIOProcessor
|
||||
|
||||
def init_io_processor(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(
|
||||
model_config=model_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.json_response_cls = get_json_response_cls()
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(*args, **kwargs)
|
||||
|
||||
async def _build_response(
|
||||
self,
|
||||
@@ -149,7 +141,7 @@ class ServingEmbedding(PoolingServing):
|
||||
data=items,
|
||||
usage=usage,
|
||||
)
|
||||
return JSONResponseCLS(content=response.model_dump())
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
def _openai_bytes_response(
|
||||
self,
|
||||
@@ -190,8 +182,8 @@ class ServingEmbedding(PoolingServing):
|
||||
media_type=response.media_type,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_cohere_response_from_ctx(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> JSONResponse:
|
||||
request = ctx.request
|
||||
@@ -218,4 +210,4 @@ class ServingEmbedding(PoolingServing):
|
||||
),
|
||||
),
|
||||
)
|
||||
return JSONResponse(content=response.model_dump(exclude_none=True))
|
||||
return self.json_response_cls(content=response.model_dump(exclude_none=True))
|
||||
|
||||
Reference in New Issue
Block a user