[Model] Deprecate the score task (this will not affect users). (#37537)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-03-20 08:07:56 +00:00
committed by GitHub
parent dcee9be95a
commit ed359c497a
22 changed files with 184 additions and 163 deletions
+2 -2
View File
@@ -1477,9 +1477,9 @@ class LLM:
data_1 = data_1 * len(data_2)
if pooling_params is None:
pooling_params = PoolingParams(task="score")
pooling_params = PoolingParams(task="classify")
elif pooling_params.task is None:
pooling_params.task = "score"
pooling_params.task = "classify"
pooling_params_list = list[PoolingParams]()
+9 -5
View File
@@ -22,7 +22,7 @@ from fastapi.middleware.cors import CORSMiddleware
from starlette.datastructures import State
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import load_chat_template
@@ -155,7 +155,9 @@ async def build_async_engine_client_from_engine_args(
def build_app(
args: Namespace, supported_tasks: tuple["SupportedTask", ...] | None = None
args: Namespace,
supported_tasks: tuple["SupportedTask", ...] | None = None,
model_config: ModelConfig | None = None,
) -> FastAPI:
if supported_tasks is None:
warnings.warn(
@@ -191,7 +193,7 @@ def build_app(
attach_router as register_sagemaker_api_router,
)
register_sagemaker_api_router(app, supported_tasks)
register_sagemaker_api_router(app, supported_tasks, model_config)
if "generate" in supported_tasks:
from vllm.entrypoints.openai.generate.api_router import (
@@ -242,7 +244,7 @@ def build_app(
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling import register_pooling_api_routers
register_pooling_api_routers(app, supported_tasks)
register_pooling_api_routers(app, supported_tasks, model_config)
app.root_path = args.root_path
app.add_middleware(
@@ -583,8 +585,10 @@ async def build_and_serve(
uvicorn_kwargs["log_config"] = log_config
supported_tasks = await engine_client.get_supported_tasks()
model_config = engine_client.model_config
logger.info("Supported tasks: %s", supported_tasks)
app = build_app(args, supported_tasks)
app = build_app(args, supported_tasks, model_config)
await init_app_state(engine_client, app.state, args, supported_tasks)
logger.info("Starting vLLM server on %s", listen_address)
+29 -11
View File
@@ -5,6 +5,9 @@ from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm.config import ModelConfig
from vllm.logger import init_logger
if TYPE_CHECKING:
from argparse import Namespace
@@ -17,9 +20,30 @@ else:
RequestLogger = object
SupportedTask = object
logger = init_logger(__name__)
def enable_scoring_api(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
) -> bool:
if any(t in supported_tasks for t in ("embed", "token_embed")):
return True
if model_config is not None and "classify" in supported_tasks:
num_labels = getattr(model_config.hf_config, "num_labels", 0)
if num_labels != 1:
logger.debug_once("Score API is only enabled for num_labels == 1.")
return False
return True
return False
def register_pooling_api_routers(
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
from vllm.entrypoints.pooling.pooling.api_router import router as pooling_router
@@ -37,11 +61,7 @@ def register_pooling_api_routers(
app.include_router(embed_router)
# Score API handles score/rerank for:
# - "score" task (score_type: cross-encoder models)
# - "embed" task (score_type: bi-encoder models)
# - "token_embed" task (score_type: late interaction models)
if any(t in supported_tasks for t in ("score", "embed", "token_embed")):
if enable_scoring_api(supported_tasks, model_config):
from vllm.entrypoints.pooling.score.api_router import router as score_router
app.include_router(score_router)
@@ -61,6 +81,8 @@ def init_pooling_state(
from vllm.entrypoints.pooling.score.serving import ServingScores
from vllm.tasks import POOLING_TASKS
model_config = engine_client.model_config
resolved_chat_template = load_chat_template(args.chat_template)
state.serving_pooling = (
@@ -102,10 +124,6 @@ def init_pooling_state(
if "classify" in supported_tasks
else None
)
# Score API handles score/rerank for:
# - "score" task (score_type: cross-encoder models)
# - "embed" task (score_type: bi-encoder models)
# - "token_embed" task (score_type: late interaction models)
state.serving_scores = (
ServingScores(
engine_client,
@@ -114,6 +132,6 @@ def init_pooling_state(
score_template=resolved_chat_template,
log_error_stack=args.log_error_stack,
)
if any(t in supported_tasks for t in ("embed", "score", "token_embed"))
if enable_scoring_api(supported_tasks, model_config)
else None
)
+2 -2
View File
@@ -35,7 +35,7 @@ class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
max_total_tokens_param="max_model_len",
)
def to_pooling_params(self, task: PoolingTask = "score"):
def to_pooling_params(self, task: PoolingTask = "classify"):
return PoolingParams(
task=task,
use_activation=self.use_activation,
@@ -111,7 +111,7 @@ class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):
max_total_tokens_param="max_model_len",
)
def to_pooling_params(self, task: PoolingTask = "score"):
def to_pooling_params(self, task: PoolingTask = "classify"):
return PoolingParams(
task=task,
use_activation=self.use_activation,
+1 -1
View File
@@ -413,7 +413,7 @@ class ServingScores(OpenAIServing):
# Schedule the request and get the result generator.
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
default_pooling_params = request.to_pooling_params("score")
default_pooling_params = request.to_pooling_params("classify")
for i, engine_prompt in enumerate(engine_prompts):
request_id_item = f"{request_id}-{i}"
+13 -5
View File
@@ -10,9 +10,11 @@ import pydantic
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling import enable_scoring_api
from vllm.entrypoints.pooling.base.serving import PoolingServing
from vllm.entrypoints.serve.instrumentator.basic import base
from vllm.entrypoints.serve.instrumentator.health import health
@@ -25,7 +27,10 @@ GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
def get_invocation_types(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
# NOTE: Items defined earlier take higher priority
INVOCATION_TYPES: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
@@ -70,7 +75,7 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
(ClassificationRequest, (classify, create_classify)),
]
if "score" in supported_tasks:
if enable_scoring_api(supported_tasks, model_config):
from vllm.entrypoints.pooling.score.api_router import do_rerank, rerank
from vllm.entrypoints.pooling.score.protocol import RerankRequest
@@ -78,7 +83,6 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
(RerankRequest, (rerank, do_rerank)),
]
if "score" in supported_tasks or "embed" in supported_tasks:
from vllm.entrypoints.pooling.score.api_router import create_score, score
from vllm.entrypoints.pooling.score.protocol import ScoreRequest
@@ -97,11 +101,15 @@ def get_invocation_types(supported_tasks: tuple["SupportedTask", ...]):
return INVOCATION_TYPES
def attach_router(app: FastAPI, supported_tasks: tuple["SupportedTask", ...]):
def attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
INVOCATION_TYPES = get_invocation_types(supported_tasks)
INVOCATION_TYPES = get_invocation_types(supported_tasks, model_config)
INVOCATION_VALIDATORS = [
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
for request_type, (get_handler, endpoint) in INVOCATION_TYPES