[Model] Deprecate the score task (this will not affect users). (#37537)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1477,9 +1477,9 @@ class LLM:
|
||||
data_1 = data_1 * len(data_2)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams(task="score")
|
||||
pooling_params = PoolingParams(task="classify")
|
||||
elif pooling_params.task is None:
|
||||
pooling_params.task = "score"
|
||||
pooling_params.task = "classify"
|
||||
|
||||
pooling_params_list = list[PoolingParams]()
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from starlette.datastructures import State
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
@@ -155,7 +155,9 @@ async def build_async_engine_client_from_engine_args(
|
||||
|
||||
|
||||
def build_app(
|
||||
args: Namespace, supported_tasks: tuple["SupportedTask", ...] | None = None
|
||||
args: Namespace,
|
||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> FastAPI:
|
||||
if supported_tasks is None:
|
||||
warnings.warn(
|
||||
@@ -191,7 +193,7 @@ def build_app(
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
register_sagemaker_api_router(app, supported_tasks)
|
||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.generate.api_router import (
|
||||
@@ -242,7 +244,7 @@ def build_app(
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling import register_pooling_api_routers
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks)
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
app.root_path = args.root_path
|
||||
app.add_middleware(
|
||||
@@ -583,8 +585,10 @@ async def build_and_serve(
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
model_config = engine_client.model_config
|
||||
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
app = build_app(args, supported_tasks)
|
||||
app = build_app(args, supported_tasks, model_config)
|
||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
@@ -5,6 +5,9 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -17,9 +20,30 @@ else:
|
||||
RequestLogger = object
|
||||
SupportedTask = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def enable_scoring_api(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> bool:
|
||||
if any(t in supported_tasks for t in ("embed", "token_embed")):
|
||||
return True
|
||||
|
||||
if model_config is not None and "classify" in supported_tasks:
|
||||
num_labels = getattr(model_config.hf_config, "num_labels", 0)
|
||||
if num_labels != 1:
|
||||
logger.debug_once("Score API is only enabled for num_labels == 1.")
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def register_pooling_api_routers(
|
||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
from vllm.entrypoints.pooling.pooling.api_router import router as pooling_router
|
||||
|
||||
@@ -37,11 +61,7 @@ def register_pooling_api_routers(
|
||||
|
||||
app.include_router(embed_router)
|
||||
|
||||
# Score API handles score/rerank for:
|
||||
# - "score" task (score_type: cross-encoder models)
|
||||
# - "embed" task (score_type: bi-encoder models)
|
||||
# - "token_embed" task (score_type: late interaction models)
|
||||
if any(t in supported_tasks for t in ("score", "embed", "token_embed")):
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from vllm.entrypoints.pooling.score.api_router import router as score_router
|
||||
|
||||
app.include_router(score_router)
|
||||
@@ -61,6 +81,8 @@ def init_pooling_state(
|
||||
from vllm.entrypoints.pooling.score.serving import ServingScores
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
|
||||
model_config = engine_client.model_config
|
||||
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
state.serving_pooling = (
|
||||
@@ -102,10 +124,6 @@ def init_pooling_state(
|
||||
if "classify" in supported_tasks
|
||||
else None
|
||||
)
|
||||
# Score API handles score/rerank for:
|
||||
# - "score" task (score_type: cross-encoder models)
|
||||
# - "embed" task (score_type: bi-encoder models)
|
||||
# - "token_embed" task (score_type: late interaction models)
|
||||
state.serving_scores = (
|
||||
ServingScores(
|
||||
engine_client,
|
||||
@@ -114,6 +132,6 @@ def init_pooling_state(
|
||||
score_template=resolved_chat_template,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
if any(t in supported_tasks for t in ("embed", "score", "token_embed"))
|
||||
if enable_scoring_api(supported_tasks, model_config)
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "score"):
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
return PoolingParams(
|
||||
task=task,
|
||||
use_activation=self.use_activation,
|
||||
@@ -111,7 +111,7 @@ class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
max_total_tokens_param="max_model_len",
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "score"):
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
return PoolingParams(
|
||||
task=task,
|
||||
use_activation=self.use_activation,
|
||||
|
||||
@@ -413,7 +413,7 @@ class ServingScores(OpenAIServing):
|
||||
# Schedule the request and get the result generator.
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
default_pooling_params = request.to_pooling_params("score")
|
||||
default_pooling_params = request.to_pooling_params("classify")
|
||||
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
request_id_item = f"{request_id}-{i}"
|
||||
|
||||
@@ -10,9 +10,11 @@ import pydantic
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.pooling import enable_scoring_api
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
@@ -25,7 +27,10 @@ GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
|
||||
def get_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
INVOCATION_TYPES: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
@@ -70,7 +75,7 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
|
||||
(ClassificationRequest, (classify, create_classify)),
|
||||
]
|
||||
|
||||
if "score" in supported_tasks:
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from vllm.entrypoints.pooling.score.api_router import do_rerank, rerank
|
||||
from vllm.entrypoints.pooling.score.protocol import RerankRequest
|
||||
|
||||
@@ -78,7 +83,6 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
|
||||
(RerankRequest, (rerank, do_rerank)),
|
||||
]
|
||||
|
||||
if "score" in supported_tasks or "embed" in supported_tasks:
|
||||
from vllm.entrypoints.pooling.score.api_router import create_score, score
|
||||
from vllm.entrypoints.pooling.score.protocol import ScoreRequest
|
||||
|
||||
@@ -97,11 +101,15 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
|
||||
return INVOCATION_TYPES
|
||||
|
||||
|
||||
def attach_router(app: FastAPI, supported_tasks: tuple["SupportedTask", ...]):
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_invocation_types(supported_tasks)
|
||||
INVOCATION_TYPES = get_invocation_types(supported_tasks, model_config)
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
|
||||
|
||||
Reference in New Issue
Block a user