[Frontend] Add detokenization streaming derender for disaggregated serving (#47301)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -5,15 +5,17 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderChatRequestUnion,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderChatStreamResponse,
|
||||
DerenderCompletionRequestUnion,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderCompletionStreamResponse,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
@@ -29,46 +31,96 @@ def derender(request: Request) -> ServingDerender | None:
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=ChatCompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
|
||||
async def derender_chat_completion(
|
||||
request: DerenderChatRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a ChatCompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path; FastAPI validates and
|
||||
routes on the ``stream`` discriminator.
|
||||
|
||||
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
|
||||
the complete token list). Returns a ``ChatCompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
|
||||
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
|
||||
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
|
||||
successive calls, one per SSE chunk from ``/inference/v1/generate``.
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(request, DerenderChatStreamRequest):
|
||||
stream_result = await handler.derender_chat_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=CompletionResponse,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
|
||||
async def derender_completion(
|
||||
request: DerenderCompletionRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a CompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path.
|
||||
|
||||
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
|
||||
``CompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderCompletionStreamRequest`` (one
|
||||
``generate_chunk`` + optional ``stream_state``). Returns a
|
||||
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(request, DerenderCompletionStreamRequest):
|
||||
stream_result = await handler.derender_completion_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderCompletionStreamResponse(
|
||||
chunk=chunk, stream_state=stream_state
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
@@ -4,8 +4,14 @@ import time
|
||||
from typing import cast
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionResponse
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
@@ -28,7 +34,10 @@ from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
@@ -200,7 +209,9 @@ class ServingDerender(BaseServing):
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses, request.prompt_tokens
|
||||
request.generate_responses,
|
||||
request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
)
|
||||
|
||||
first = request.generate_responses[0]
|
||||
@@ -237,6 +248,90 @@ class ServingDerender(BaseServing):
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
async def derender_chat_stream_response(
|
||||
self,
|
||||
request: DerenderChatStreamRequest,
|
||||
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_chat_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk and returns the
|
||||
derendered chunk together with the updated client carried state.
|
||||
|
||||
``parser is None`` or no ``chat_request`` until reasoning/tool call
|
||||
functionality added in future PR.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
try:
|
||||
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
|
||||
model=request.model,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
chat_request=request.chat_request,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
)
|
||||
except NotImplementedError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
request.model,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
async def derender_completion_stream_response(
|
||||
self,
|
||||
request: DerenderCompletionStreamRequest,
|
||||
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_completion_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
|
||||
delta) and returns the derendered chunk and updated state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
try:
|
||||
(
|
||||
chunk,
|
||||
updated_state,
|
||||
) = await self.online_derenderer.derender_completion_stream(
|
||||
model=request.model,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
request.model,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
|
||||
Reference in New Issue
Block a user