[Frontend] Resettle generative scoring entrypoint. (#44153)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-01 07:54:59 +00:00
committed by GitHub
parent 1f6048abe5
commit 0910f7e0e1
13 changed files with 38 additions and 58 deletions
+5 -1
View File
@@ -1275,10 +1275,12 @@ steps:
- vllm/ - vllm/
- tests/entrypoints/openai - tests/entrypoints/openai
- tests/entrypoints/test_chat_utils - tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py - pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s entrypoints/generate
- label: Entrypoints Integration (API Server openai - Part 3) # TBD - label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
@@ -1368,7 +1370,7 @@ steps:
- vllm/platforms/rocm.py - vllm/platforms/rocm.py
commands: commands:
- pytest -v -s entrypoints/openai/tool_parsers - pytest -v -s entrypoints/openai/tool_parsers
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
- label: OpenAI API correctness # TBD - label: OpenAI API correctness # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
@@ -2782,10 +2784,12 @@ steps:
- vllm/ - vllm/
- tests/entrypoints/openai - tests/entrypoints/openai
- tests/entrypoints/test_chat_utils - tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py - pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s entrypoints/generate
- label: Entrypoints Integration (API Server openai - Part 3) # TBD - label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
+3 -1
View File
@@ -11,7 +11,7 @@ steps:
- tests/entrypoints/ - tests/entrypoints/
commands: commands:
- pytest -v -s entrypoints/openai/tool_parsers - pytest -v -s entrypoints/openai/tool_parsers
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
- label: Entrypoints Integration (LLM) - label: Entrypoints Integration (LLM)
key: entrypoints-integration-llm key: entrypoints-integration-llm
@@ -60,9 +60,11 @@ steps:
- vllm/ - vllm/
- tests/entrypoints/openai - tests/entrypoints/openai
- tests/entrypoints/test_chat_utils - tests/entrypoints/test_chat_utils
- tests/entrypoints/generate
commands: commands:
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/test_chat_utils.py - pytest -v -s entrypoints/test_chat_utils.py
- pytest -v -s entrypoints/generate
mirror: mirror:
amd: amd:
device: mi325_1 device: mi325_1
@@ -18,13 +18,13 @@ from unittest.mock import MagicMock
import pytest import pytest
from vllm.config.multimodal import MultiModalConfig from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.generate.generative_scoring.serving import (
from vllm.entrypoints.openai.generative_scoring.serving import (
GenerativeScoringItemResult, GenerativeScoringItemResult,
GenerativeScoringRequest, GenerativeScoringRequest,
GenerativeScoringResponse, GenerativeScoringResponse,
OpenAIServingGenerativeScoring, ServingGenerativeScoring,
) )
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.logprobs import Logprob from vllm.logprobs import Logprob
@@ -86,13 +86,13 @@ def _create_mock_engine():
return mock_engine return mock_engine
def _create_serving(mock_engine) -> OpenAIServingGenerativeScoring: def _create_serving(mock_engine) -> ServingGenerativeScoring:
"""Create an OpenAIServingGenerativeScoring instance with mocks.""" """Create an ServingGenerativeScoring instance with mocks."""
models = OpenAIServingModels( models = OpenAIServingModels(
engine_client=mock_engine, engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS, base_model_paths=BASE_MODEL_PATHS,
) )
return OpenAIServingGenerativeScoring(mock_engine, models, request_logger=None) return ServingGenerativeScoring(mock_engine, models, request_logger=None)
def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput: def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput:
@@ -186,7 +186,7 @@ class TestProbabilityComputation:
self, label_logprobs, apply_softmax, should_sum_to_one self, label_logprobs, apply_softmax, should_sum_to_one
): ):
"""Test probability computation for softmax and true probability modes.""" """Test probability computation for softmax and true probability modes."""
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring) serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
probs = serving._compute_probabilities( probs = serving._compute_probabilities(
label_logprobs, apply_softmax=apply_softmax label_logprobs, apply_softmax=apply_softmax
) )
@@ -211,7 +211,7 @@ class TestProbabilityComputation:
def test_score_formula(self): def test_score_formula(self):
"""Test the score formula: P(token[0]) / (P(token[0]) + P(token[1])).""" """Test the score formula: P(token[0]) / (P(token[0]) + P(token[1]))."""
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring) serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
# With logprobs -0.5 and -2.0, softmax gives higher prob to first token # With logprobs -0.5 and -2.0, softmax gives higher prob to first token
logprobs = {9454: -0.5, 2753: -2.0} logprobs = {9454: -0.5, 2753: -2.0}
@@ -8,7 +8,7 @@ Tests verify the full HTTP request/response flow using RemoteOpenAIServer.
import pytest import pytest
import requests import requests
from ....utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B" MODEL_NAME = "Qwen/Qwen3-0.6B"
@@ -41,6 +41,10 @@ def register_generate_api_routers(app: FastAPI):
register_anthropic_api_router(app) register_anthropic_api_router(app)
from .generative_scoring.api_router import register_generative_scoring_api_router
register_generative_scoring_api_router(app)
async def init_generate_state( async def init_generate_state(
engine_client: "EngineClient", engine_client: "EngineClient",
@@ -185,3 +189,11 @@ async def init_generate_state(
if "generate" in supported_tasks if "generate" in supported_tasks
else None else None
) )
from .generative_scoring.serving import ServingGenerativeScoring
state.serving_generative_scoring = ServingGenerativeScoring(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
@@ -1,34 +1,25 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus from http import HTTPStatus
from typing import TYPE_CHECKING
from fastapi import APIRouter, Depends, FastAPI, Request from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.generate.generative_scoring.serving import (
from vllm.entrypoints.openai.generative_scoring.serving import (
GenerativeScoringResponse, GenerativeScoringResponse,
OpenAIServingGenerativeScoring, ServingGenerativeScoring,
) )
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.utils import validate_json_request from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.utils import load_aware_call, with_cancellation from vllm.entrypoints.utils import load_aware_call, with_cancellation
from vllm.logger import init_logger from vllm.logger import init_logger
if TYPE_CHECKING:
from argparse import Namespace
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
router = APIRouter() router = APIRouter()
logger = init_logger(__name__) logger = init_logger(__name__)
def generative_scoring(request: Request) -> OpenAIServingGenerativeScoring | None: def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
return request.app.state.serving_generative_scoring return request.app.state.serving_generative_scoring
@@ -51,7 +42,7 @@ async def create_generative_scoring(raw_request: Request):
raw_body = await raw_request.json() raw_body = await raw_request.json()
from vllm.entrypoints.openai.generative_scoring.serving import ( from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringRequest, GenerativeScoringRequest,
) )
@@ -68,20 +59,3 @@ async def create_generative_scoring(raw_request: Request):
def register_generative_scoring_api_router(app: FastAPI): def register_generative_scoring_api_router(app: FastAPI):
app.include_router(router) app.include_router(router)
async def init_generative_scoring_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: "RequestLogger | None",
):
from vllm.entrypoints.openai.generative_scoring.serving import (
OpenAIServingGenerativeScoring,
)
state.serving_generative_scoring = OpenAIServingGenerativeScoring(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
# ============================================================================ # ============================================================================
class OpenAIServingGenerativeScoring(OpenAIServing): class ServingGenerativeScoring(OpenAIServing):
"""Serving class for generative scoring computation. """Serving class for generative scoring computation.
This class handles computing the probability of specified token IDs This class handles computing the probability of specified token IDs
+2 -14
View File
@@ -196,7 +196,7 @@ def build_app(
register_sagemaker_api_router(app, supported_tasks, model_config) register_sagemaker_api_router(app, supported_tasks, model_config)
if "generate" in supported_tasks: if "generate" in supported_tasks:
from vllm.entrypoints.openai.generate.api_router import ( from vllm.entrypoints.generate.api_router import (
register_generate_api_routers, register_generate_api_routers,
) )
@@ -220,12 +220,6 @@ def build_app(
elastic_ep_attach_router(app) elastic_ep_attach_router(app)
from vllm.entrypoints.openai.generative_scoring.api_router import (
register_generative_scoring_api_router,
)
register_generative_scoring_api_router(app)
if "generate" in supported_tasks or "render" in supported_tasks: if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.serve.render.api_router import ( from vllm.entrypoints.serve.render.api_router import (
attach_router as attach_render_router, attach_router as attach_render_router,
@@ -402,18 +396,12 @@ async def init_app_state(
) )
if "generate" in supported_tasks: if "generate" in supported_tasks:
from vllm.entrypoints.openai.generate.api_router import init_generate_state from vllm.entrypoints.generate.api_router import init_generate_state
await init_generate_state( await init_generate_state(
engine_client, state, args, request_logger, supported_tasks engine_client, state, args, request_logger, supported_tasks
) )
from vllm.entrypoints.openai.generative_scoring.api_router import (
init_generative_scoring_state,
)
await init_generative_scoring_state(engine_client, state, args, request_logger)
if "transcription" in supported_tasks or "realtime" in supported_tasks: if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
+1 -1
View File
@@ -11,9 +11,9 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
from vllm.entrypoints.openai.utils import validate_json_request from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling.base.serving import PoolingServingBase from vllm.entrypoints.pooling.base.serving import PoolingServingBase
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types from vllm.entrypoints.pooling.factories import get_pooling_invocation_types