[Frontend] Frontend will only attach supported tasks corresponding entrypoints. (#33139)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -11,40 +11,54 @@ if TYPE_CHECKING:
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
else:
|
||||
RequestLogger = object
|
||||
SupportedTask = object
|
||||
|
||||
|
||||
def register_pooling_api_routers(app: FastAPI):
|
||||
from vllm.entrypoints.pooling.classify.api_router import router as classify_router
|
||||
from vllm.entrypoints.pooling.embed.api_router import router as embed_router
|
||||
def register_pooling_api_routers(
|
||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||
):
|
||||
from vllm.entrypoints.pooling.pooling.api_router import router as pooling_router
|
||||
from vllm.entrypoints.pooling.score.api_router import router as score_router
|
||||
|
||||
app.include_router(classify_router)
|
||||
app.include_router(embed_router)
|
||||
app.include_router(score_router)
|
||||
app.include_router(pooling_router)
|
||||
|
||||
if "classify" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.classify.api_router import (
|
||||
router as classify_router,
|
||||
)
|
||||
|
||||
async def init_pooling_state(
|
||||
engine_client: "EngineClient", state: "State", args: "Namespace"
|
||||
app.include_router(classify_router)
|
||||
|
||||
if "embed" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.embed.api_router import router as embed_router
|
||||
|
||||
app.include_router(embed_router)
|
||||
|
||||
if "score" in supported_tasks or "embed" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.score.api_router import router as score_router
|
||||
|
||||
app.include_router(score_router)
|
||||
|
||||
|
||||
def init_pooling_state(
|
||||
engine_client: "EngineClient",
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.pooling.classify.serving import ServingClassification
|
||||
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
|
||||
from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling
|
||||
from vllm.entrypoints.pooling.score.serving import ServingScores
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
state.openai_serving_pooling = (
|
||||
(
|
||||
OpenAIServingPooling(
|
||||
|
||||
Reference in New Issue
Block a user