[Frontend][last/5] Improve pooling entrypoints | clean up. (#39675)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
5e5afafa21
commit
4e8c3f1c19
@@ -13,12 +13,13 @@ from fastapi.responses import JSONResponse, Response
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
||||
from vllm.entrypoints.pooling.utils import enable_scoring_api
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
@@ -27,80 +28,6 @@ GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServingBase | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def get_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
INVOCATION_TYPES: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||
chat,
|
||||
create_chat_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.api_router import (
|
||||
completion,
|
||||
create_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(ChatCompletionRequest, (chat, create_chat_completion)),
|
||||
(CompletionRequest, (completion, create_completion)),
|
||||
]
|
||||
|
||||
if "embed" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.embed.api_router import (
|
||||
create_embedding,
|
||||
embedding,
|
||||
)
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(EmbeddingRequest, (embedding, create_embedding)),
|
||||
]
|
||||
|
||||
if "classify" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.classify.api_router import (
|
||||
classify,
|
||||
create_classify,
|
||||
)
|
||||
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(ClassificationRequest, (classify, create_classify)),
|
||||
]
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from vllm.entrypoints.pooling.scoring.api_router import do_rerank, rerank
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(RerankRequest, (rerank, do_rerank)),
|
||||
]
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.api_router import create_score, score
|
||||
from vllm.entrypoints.pooling.scoring.protocol import ScoreRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(ScoreRequest, (score, create_score)),
|
||||
]
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.pooling.api_router import create_pooling, pooling
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingRequest
|
||||
|
||||
INVOCATION_TYPES += [
|
||||
(PoolingRequest, (pooling, create_pooling)),
|
||||
]
|
||||
|
||||
return INVOCATION_TYPES
|
||||
|
||||
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
@@ -109,7 +36,10 @@ def attach_router(
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_invocation_types(supported_tasks, model_config)
|
||||
INVOCATION_TYPES = get_generate_invocation_types(
|
||||
supported_tasks, model_config
|
||||
) + get_pooling_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
|
||||
|
||||
Reference in New Issue
Block a user