wang.yuqiandCyrus Leung
eeb549a74d
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
98 lines
3.3 KiB
Python
98 lines
3.3 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from collections.abc import AsyncGenerator
|
|
|
|
from fastapi import Request
|
|
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
|
from vllm.logger import init_logger
|
|
from vllm.outputs import RequestOutput
|
|
|
|
from ..base.serving import SpeechToTextBaseServing
|
|
from .protocol import (
|
|
TranslationRequest,
|
|
TranslationResponse,
|
|
TranslationResponseStreamChoice,
|
|
TranslationResponseVerbose,
|
|
TranslationStreamResponse,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class OpenAIServingTranslation(SpeechToTextBaseServing):
|
|
"""Handles translation requests."""
|
|
|
|
def __init__(
|
|
self,
|
|
engine_client: EngineClient,
|
|
models: OpenAIServingModels,
|
|
*,
|
|
request_logger: RequestLogger | None,
|
|
return_tokens_as_token_ids: bool = False,
|
|
enable_force_include_usage: bool = False,
|
|
):
|
|
super().__init__(
|
|
engine_client=engine_client,
|
|
models=models,
|
|
request_logger=request_logger,
|
|
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
|
task_type="translate",
|
|
enable_force_include_usage=enable_force_include_usage,
|
|
)
|
|
|
|
async def create_translation(
|
|
self,
|
|
audio_data: bytes,
|
|
request: TranslationRequest,
|
|
raw_request: Request | None = None,
|
|
) -> (
|
|
TranslationResponse
|
|
| TranslationResponseVerbose
|
|
| AsyncGenerator[str, None]
|
|
| ErrorResponse
|
|
):
|
|
"""Translation API similar to OpenAI's API.
|
|
|
|
See https://platform.openai.com/docs/api-reference/audio/createTranslation
|
|
for the API specification. This API mimics the OpenAI translation API.
|
|
"""
|
|
return await self._create_speech_to_text(
|
|
audio_data=audio_data,
|
|
request=request,
|
|
raw_request=raw_request,
|
|
response_class=(
|
|
TranslationResponseVerbose
|
|
if request.response_format == "verbose_json"
|
|
else TranslationResponse
|
|
),
|
|
stream_generator_method=self.translation_stream_generator,
|
|
)
|
|
|
|
async def translation_stream_generator(
|
|
self,
|
|
request: TranslationRequest,
|
|
result_generator: list[AsyncGenerator[RequestOutput, None]],
|
|
request_id: str,
|
|
request_metadata: RequestResponseMetadata,
|
|
audio_duration_s: float,
|
|
separator: str,
|
|
) -> AsyncGenerator[str, None]:
|
|
generator = self._speech_to_text_stream_generator(
|
|
request=request,
|
|
list_result_generator=result_generator,
|
|
request_id=request_id,
|
|
request_metadata=request_metadata,
|
|
audio_duration_s=audio_duration_s,
|
|
chunk_object_type="translation.chunk",
|
|
response_stream_choice_class=TranslationResponseStreamChoice,
|
|
stream_response_class=TranslationStreamResponse,
|
|
separator=separator,
|
|
)
|
|
async for chunk in generator:
|
|
yield chunk
|