[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -5,15 +5,17 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderChatRequestUnion,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderChatStreamResponse,
|
||||
DerenderCompletionRequestUnion,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderCompletionStreamResponse,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
@@ -29,46 +31,96 @@ def derender(request: Request) -> ServingDerender | None:
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=ChatCompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||
async def derender_chat_completion(
|
||||
request: DerenderChatRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a ChatCompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path; FastAPI validates and
|
||||
routes on the ``stream`` discriminator.
|
||||
|
||||
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
|
||||
the complete token list). Returns a ``ChatCompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
|
||||
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
|
||||
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
|
||||
successive calls, one per SSE chunk from ``/inference/v1/generate``.
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(request, DerenderChatStreamRequest):
|
||||
stream_result = await handler.derender_chat_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=CompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||
async def derender_completion(
|
||||
request: DerenderCompletionRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a CompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path.
|
||||
|
||||
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
|
||||
``CompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderCompletionStreamRequest`` (one
|
||||
``generate_chunk`` + optional ``stream_state``). Returns a
|
||||
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(request, DerenderCompletionStreamRequest):
|
||||
stream_result = await handler.derender_completion_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderCompletionStreamResponse(
|
||||
chunk=chunk, stream_state=stream_state
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
Reference in New Issue
Block a user