[Frontend] feat: add streaming support for token generation endpoint (#37171)
Signed-off-by: Hyeonki Hong <hyeonki.hong@moreh.io>
This commit is contained in:
@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions, UsageInfo
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -122,6 +122,26 @@ class GenerateResponseChoice(BaseModel):
|
||||
token_ids: list[int] | None = None
|
||||
|
||||
|
||||
class GenerateResponseStreamChoice(BaseModel):
|
||||
index: int
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
finish_reason: str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
|
||||
|
||||
class GenerateStreamResponse(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
choices: list[GenerateResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
PromptTokenUsageInfo,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
@@ -28,12 +29,15 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.utils import should_include_usage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils.collection_utils import as_list
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -74,7 +78,7 @@ class ServingTokens(OpenAIServing):
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateResponse | ErrorResponse:
|
||||
) -> GenerateResponse | ErrorResponse | AsyncGenerator[str, None]:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
@@ -110,6 +114,8 @@ class ServingTokens(OpenAIServing):
|
||||
sampling_params = request.sampling_params
|
||||
if self.force_no_detokenize:
|
||||
sampling_params.detokenize = False
|
||||
if request.stream:
|
||||
sampling_params.output_kind = RequestOutputKind.DELTA
|
||||
|
||||
self._log_inputs(
|
||||
request_id,
|
||||
@@ -133,9 +139,17 @@ class ServingTokens(OpenAIServing):
|
||||
priority=request.priority,
|
||||
)
|
||||
|
||||
# TODO(NickLucche): Implement streaming response
|
||||
|
||||
assert result_generator is not None
|
||||
|
||||
if request.stream:
|
||||
return self.serve_tokens_stream_generator(
|
||||
request,
|
||||
result_generator,
|
||||
request_id,
|
||||
model_name,
|
||||
request_metadata,
|
||||
)
|
||||
|
||||
return await self.serve_tokens_full_generator(
|
||||
request, result_generator, request_id, model_name, request_metadata
|
||||
)
|
||||
@@ -236,6 +250,109 @@ class ServingTokens(OpenAIServing):
|
||||
|
||||
return response
|
||||
|
||||
async def serve_tokens_stream_generator(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
result_generator: AsyncGenerator[RequestOutput, None],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
num_prompt_tokens = 0
|
||||
num_generated_tokens: list[int] = []
|
||||
first_iteration = True
|
||||
num_cached_tokens = None
|
||||
sampling_params: SamplingParams = request.sampling_params
|
||||
|
||||
include_usage, include_continuous_usage = should_include_usage(
|
||||
request.stream_options, False
|
||||
)
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
if first_iteration:
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if res.encoder_prompt_token_ids is not None:
|
||||
num_prompt_tokens += len(res.encoder_prompt_token_ids)
|
||||
num_cached_tokens = res.num_cached_tokens
|
||||
num_generated_tokens = [0] * len(res.outputs)
|
||||
first_iteration = False
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index
|
||||
delta_token_ids = output.token_ids
|
||||
num_generated_tokens[i] += len(delta_token_ids)
|
||||
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
if not delta_token_ids:
|
||||
continue
|
||||
|
||||
if sampling_params.logprobs is not None:
|
||||
out_logprobs = output.logprobs
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
)
|
||||
],
|
||||
)
|
||||
if include_continuous_usage:
|
||||
chunk.usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=num_generated_tokens[i],
|
||||
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
|
||||
)
|
||||
|
||||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||||
|
||||
total_completion_tokens = sum(num_generated_tokens)
|
||||
final_usage_info = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=num_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
final_chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=final_usage_info,
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
request_metadata.final_usage_info = final_usage_info
|
||||
|
||||
except GenerationError as e:
|
||||
yield (
|
||||
f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error in token generation stream.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
def _create_tokens_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
|
||||
Reference in New Issue
Block a user