[Frontend] Resettle generative scoring entrypoint. (#44153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
+12
@@ -41,6 +41,10 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
|
||||
|
||||
async def init_generate_state(
|
||||
engine_client: "EngineClient",
|
||||
@@ -185,3 +189,11 @@ async def init_generate_state(
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
state.serving_generative_scoring = ServingGenerativeScoring(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
+5
-31
@@ -1,34 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringResponse,
|
||||
OpenAIServingGenerativeScoring,
|
||||
ServingGenerativeScoring,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.utils import load_aware_call, with_cancellation
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generative_scoring(request: Request) -> OpenAIServingGenerativeScoring | None:
|
||||
def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
|
||||
return request.app.state.serving_generative_scoring
|
||||
|
||||
|
||||
@@ -51,7 +42,7 @@ async def create_generative_scoring(raw_request: Request):
|
||||
|
||||
raw_body = await raw_request.json()
|
||||
|
||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
)
|
||||
|
||||
@@ -68,20 +59,3 @@ async def create_generative_scoring(raw_request: Request):
|
||||
|
||||
def register_generative_scoring_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
async def init_generative_scoring_state(
|
||||
engine_client: "EngineClient",
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
request_logger: "RequestLogger | None",
|
||||
):
|
||||
from vllm.entrypoints.openai.generative_scoring.serving import (
|
||||
OpenAIServingGenerativeScoring,
|
||||
)
|
||||
|
||||
state.serving_generative_scoring = OpenAIServingGenerativeScoring(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
+1
-1
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class OpenAIServingGenerativeScoring(OpenAIServing):
|
||||
class ServingGenerativeScoring(OpenAIServing):
|
||||
"""Serving class for generative scoring computation.
|
||||
|
||||
This class handles computing the probability of specified token IDs
|
||||
@@ -196,7 +196,7 @@ def build_app(
|
||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.generate.api_router import (
|
||||
from vllm.entrypoints.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
)
|
||||
|
||||
@@ -220,12 +220,6 @@ def build_app(
|
||||
|
||||
elastic_ep_attach_router(app)
|
||||
|
||||
from vllm.entrypoints.openai.generative_scoring.api_router import (
|
||||
register_generative_scoring_api_router,
|
||||
)
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
|
||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||
from vllm.entrypoints.serve.render.api_router import (
|
||||
attach_router as attach_render_router,
|
||||
@@ -402,18 +396,12 @@ async def init_app_state(
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.generate.api_router import init_generate_state
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
from vllm.entrypoints.openai.generative_scoring.api_router import (
|
||||
init_generative_scoring_state,
|
||||
)
|
||||
|
||||
await init_generative_scoring_state(engine_client, state, args, request_logger)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
|
||||
Reference in New Issue
Block a user