[Bugfix] Fix generative scoring body cancellation (#57729)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
234df712ed
commit
50f78084da
@@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
GenerativeScoringResponse,
|
||||
ServingGenerativeScoring,
|
||||
)
|
||||
@@ -36,21 +37,16 @@ def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_generative_scoring(raw_request: Request):
|
||||
async def create_generative_scoring(
|
||||
request: GenerativeScoringRequest, raw_request: Request
|
||||
):
|
||||
handler = generative_scoring(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the Generative Scoring API"
|
||||
)
|
||||
|
||||
raw_body = await raw_request.json()
|
||||
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
)
|
||||
|
||||
gen_request = GenerativeScoringRequest(**raw_body)
|
||||
result = await handler.create_generative_scoring(gen_request, raw_request)
|
||||
result = await handler.create_generative_scoring(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
Reference in New Issue
Block a user