[Frontend] Resettle generative scoring entrypoint. (#44153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1275,10 +1275,12 @@ steps:
|
|||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/openai
|
- tests/entrypoints/openai
|
||||||
- tests/entrypoints/test_chat_utils
|
- tests/entrypoints/test_chat_utils
|
||||||
|
- tests/entrypoints/generate
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||||
- pytest -v -s entrypoints/test_chat_utils.py
|
- pytest -v -s entrypoints/test_chat_utils.py
|
||||||
|
- pytest -v -s entrypoints/generate
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
|
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -1368,7 +1370,7 @@ steps:
|
|||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/openai/tool_parsers
|
- pytest -v -s entrypoints/openai/tool_parsers
|
||||||
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
|
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
|
||||||
|
|
||||||
- label: OpenAI API correctness # TBD
|
- label: OpenAI API correctness # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -2782,10 +2784,12 @@ steps:
|
|||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/openai
|
- tests/entrypoints/openai
|
||||||
- tests/entrypoints/test_chat_utils
|
- tests/entrypoints/test_chat_utils
|
||||||
|
- tests/entrypoints/generate
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||||
- pytest -v -s entrypoints/test_chat_utils.py
|
- pytest -v -s entrypoints/test_chat_utils.py
|
||||||
|
- pytest -v -s entrypoints/generate
|
||||||
|
|
||||||
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
|
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ steps:
|
|||||||
- tests/entrypoints/
|
- tests/entrypoints/
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/openai/tool_parsers
|
- pytest -v -s entrypoints/openai/tool_parsers
|
||||||
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text
|
- pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
|
||||||
|
|
||||||
- label: Entrypoints Integration (LLM)
|
- label: Entrypoints Integration (LLM)
|
||||||
key: entrypoints-integration-llm
|
key: entrypoints-integration-llm
|
||||||
@@ -60,9 +60,11 @@ steps:
|
|||||||
- vllm/
|
- vllm/
|
||||||
- tests/entrypoints/openai
|
- tests/entrypoints/openai
|
||||||
- tests/entrypoints/test_chat_utils
|
- tests/entrypoints/test_chat_utils
|
||||||
|
- tests/entrypoints/generate
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||||
- pytest -v -s entrypoints/test_chat_utils.py
|
- pytest -v -s entrypoints/test_chat_utils.py
|
||||||
|
- pytest -v -s entrypoints/generate
|
||||||
mirror:
|
mirror:
|
||||||
amd:
|
amd:
|
||||||
device: mi325_1
|
device: mi325_1
|
||||||
|
|||||||
+8
-8
@@ -18,13 +18,13 @@ from unittest.mock import MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config.multimodal import MultiModalConfig
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
|
||||||
GenerativeScoringItemResult,
|
GenerativeScoringItemResult,
|
||||||
GenerativeScoringRequest,
|
GenerativeScoringRequest,
|
||||||
GenerativeScoringResponse,
|
GenerativeScoringResponse,
|
||||||
OpenAIServingGenerativeScoring,
|
ServingGenerativeScoring,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
@@ -86,13 +86,13 @@ def _create_mock_engine():
|
|||||||
return mock_engine
|
return mock_engine
|
||||||
|
|
||||||
|
|
||||||
def _create_serving(mock_engine) -> OpenAIServingGenerativeScoring:
|
def _create_serving(mock_engine) -> ServingGenerativeScoring:
|
||||||
"""Create an OpenAIServingGenerativeScoring instance with mocks."""
|
"""Create an ServingGenerativeScoring instance with mocks."""
|
||||||
models = OpenAIServingModels(
|
models = OpenAIServingModels(
|
||||||
engine_client=mock_engine,
|
engine_client=mock_engine,
|
||||||
base_model_paths=BASE_MODEL_PATHS,
|
base_model_paths=BASE_MODEL_PATHS,
|
||||||
)
|
)
|
||||||
return OpenAIServingGenerativeScoring(mock_engine, models, request_logger=None)
|
return ServingGenerativeScoring(mock_engine, models, request_logger=None)
|
||||||
|
|
||||||
|
|
||||||
def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput:
|
def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput:
|
||||||
@@ -186,7 +186,7 @@ class TestProbabilityComputation:
|
|||||||
self, label_logprobs, apply_softmax, should_sum_to_one
|
self, label_logprobs, apply_softmax, should_sum_to_one
|
||||||
):
|
):
|
||||||
"""Test probability computation for softmax and true probability modes."""
|
"""Test probability computation for softmax and true probability modes."""
|
||||||
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
|
serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
|
||||||
probs = serving._compute_probabilities(
|
probs = serving._compute_probabilities(
|
||||||
label_logprobs, apply_softmax=apply_softmax
|
label_logprobs, apply_softmax=apply_softmax
|
||||||
)
|
)
|
||||||
@@ -211,7 +211,7 @@ class TestProbabilityComputation:
|
|||||||
|
|
||||||
def test_score_formula(self):
|
def test_score_formula(self):
|
||||||
"""Test the score formula: P(token[0]) / (P(token[0]) + P(token[1]))."""
|
"""Test the score formula: P(token[0]) / (P(token[0]) + P(token[1]))."""
|
||||||
serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
|
serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
|
||||||
|
|
||||||
# With logprobs -0.5 and -2.0, softmax gives higher prob to first token
|
# With logprobs -0.5 and -2.0, softmax gives higher prob to first token
|
||||||
logprobs = {9454: -0.5, 2753: -2.0}
|
logprobs = {9454: -0.5, 2753: -2.0}
|
||||||
+1
-1
@@ -8,7 +8,7 @@ Tests verify the full HTTP request/response flow using RemoteOpenAIServer.
|
|||||||
import pytest
|
import pytest
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from ....utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
|
|
||||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||||
|
|
||||||
+12
@@ -41,6 +41,10 @@ def register_generate_api_routers(app: FastAPI):
|
|||||||
|
|
||||||
register_anthropic_api_router(app)
|
register_anthropic_api_router(app)
|
||||||
|
|
||||||
|
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||||
|
|
||||||
|
register_generative_scoring_api_router(app)
|
||||||
|
|
||||||
|
|
||||||
async def init_generate_state(
|
async def init_generate_state(
|
||||||
engine_client: "EngineClient",
|
engine_client: "EngineClient",
|
||||||
@@ -185,3 +189,11 @@ async def init_generate_state(
|
|||||||
if "generate" in supported_tasks
|
if "generate" in supported_tasks
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from .generative_scoring.serving import ServingGenerativeScoring
|
||||||
|
|
||||||
|
state.serving_generative_scoring = ServingGenerativeScoring(
|
||||||
|
engine_client,
|
||||||
|
state.openai_serving_models,
|
||||||
|
request_logger=request_logger,
|
||||||
|
)
|
||||||
+5
-31
@@ -1,34 +1,25 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
|
||||||
GenerativeScoringResponse,
|
GenerativeScoringResponse,
|
||||||
OpenAIServingGenerativeScoring,
|
ServingGenerativeScoring,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.utils import load_aware_call, with_cancellation
|
from vllm.entrypoints.utils import load_aware_call, with_cancellation
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from argparse import Namespace
|
|
||||||
|
|
||||||
from starlette.datastructures import State
|
|
||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
|
||||||
from vllm.entrypoints.logger import RequestLogger
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def generative_scoring(request: Request) -> OpenAIServingGenerativeScoring | None:
|
def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
|
||||||
return request.app.state.serving_generative_scoring
|
return request.app.state.serving_generative_scoring
|
||||||
|
|
||||||
|
|
||||||
@@ -51,7 +42,7 @@ async def create_generative_scoring(raw_request: Request):
|
|||||||
|
|
||||||
raw_body = await raw_request.json()
|
raw_body = await raw_request.json()
|
||||||
|
|
||||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||||
GenerativeScoringRequest,
|
GenerativeScoringRequest,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -68,20 +59,3 @@ async def create_generative_scoring(raw_request: Request):
|
|||||||
|
|
||||||
def register_generative_scoring_api_router(app: FastAPI):
|
def register_generative_scoring_api_router(app: FastAPI):
|
||||||
app.include_router(router)
|
app.include_router(router)
|
||||||
|
|
||||||
|
|
||||||
async def init_generative_scoring_state(
|
|
||||||
engine_client: "EngineClient",
|
|
||||||
state: "State",
|
|
||||||
args: "Namespace",
|
|
||||||
request_logger: "RequestLogger | None",
|
|
||||||
):
|
|
||||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
|
||||||
OpenAIServingGenerativeScoring,
|
|
||||||
)
|
|
||||||
|
|
||||||
state.serving_generative_scoring = OpenAIServingGenerativeScoring(
|
|
||||||
engine_client,
|
|
||||||
state.openai_serving_models,
|
|
||||||
request_logger=request_logger,
|
|
||||||
)
|
|
||||||
+1
-1
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
|
|||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
|
|
||||||
class OpenAIServingGenerativeScoring(OpenAIServing):
|
class ServingGenerativeScoring(OpenAIServing):
|
||||||
"""Serving class for generative scoring computation.
|
"""Serving class for generative scoring computation.
|
||||||
|
|
||||||
This class handles computing the probability of specified token IDs
|
This class handles computing the probability of specified token IDs
|
||||||
@@ -196,7 +196,7 @@ def build_app(
|
|||||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||||
|
|
||||||
if "generate" in supported_tasks:
|
if "generate" in supported_tasks:
|
||||||
from vllm.entrypoints.openai.generate.api_router import (
|
from vllm.entrypoints.generate.api_router import (
|
||||||
register_generate_api_routers,
|
register_generate_api_routers,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -220,12 +220,6 @@ def build_app(
|
|||||||
|
|
||||||
elastic_ep_attach_router(app)
|
elastic_ep_attach_router(app)
|
||||||
|
|
||||||
from vllm.entrypoints.openai.generative_scoring.api_router import (
|
|
||||||
register_generative_scoring_api_router,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_generative_scoring_api_router(app)
|
|
||||||
|
|
||||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||||
from vllm.entrypoints.serve.render.api_router import (
|
from vllm.entrypoints.serve.render.api_router import (
|
||||||
attach_router as attach_render_router,
|
attach_router as attach_render_router,
|
||||||
@@ -402,18 +396,12 @@ async def init_app_state(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if "generate" in supported_tasks:
|
if "generate" in supported_tasks:
|
||||||
from vllm.entrypoints.openai.generate.api_router import init_generate_state
|
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||||
|
|
||||||
await init_generate_state(
|
await init_generate_state(
|
||||||
engine_client, state, args, request_logger, supported_tasks
|
engine_client, state, args, request_logger, supported_tasks
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.openai.generative_scoring.api_router import (
|
|
||||||
init_generative_scoring_state,
|
|
||||||
)
|
|
||||||
|
|
||||||
await init_generative_scoring_state(engine_client, state, args, request_logger)
|
|
||||||
|
|
||||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||||
|
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
|||||||
from fastapi.responses import JSONResponse, Response
|
from fastapi.responses import JSONResponse, Response
|
||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||||
from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
|
|
||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
||||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||||
|
|||||||
Reference in New Issue
Block a user