[Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543)

Signed-off-by: Lucas Christian <lucaschgf7@gmail.com>
This commit is contained in:
christian
2026-07-29 23:24:30 +00:00
committed by GitHub
parent 1cb3fe5844
commit 9d1aa4dda8
9 changed files with 309 additions and 10 deletions
@@ -8,4 +8,5 @@ import torch
## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
_LONG_INFO = torch.iinfo(torch.long)
@@ -43,6 +43,7 @@ from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterat
from ..transcription.protocol import (
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionSegment,
@@ -56,7 +57,9 @@ from ..translation.protocol import (
TranslationStreamResponse,
)
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponse: TypeAlias = (
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
)
SpeechToTextResponseVerbose: TypeAlias = (
TranscriptionResponseVerbose | TranslationResponseVerbose
)
@@ -70,6 +73,7 @@ ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| TranslationResponseVerbose
)
@@ -448,10 +452,15 @@ class SpeechToTextBaseServing(GenerateBaseServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
if request.response_format not in ["text", "json", "verbose_json"]:
if request.response_format not in [
"text",
"json",
"verbose_json",
"diarized_json",
]:
return self.create_error_response(
"Currently only support response_format: "
"`text`, `json` or `verbose_json`"
"`text`, `json`, `verbose_json` or `diarized_json`"
)
if (
@@ -462,9 +471,20 @@ class SpeechToTextBaseServing(GenerateBaseServing):
f"Currently do not support verbose_json for {request.model}"
)
if request.response_format == "verbose_json" and request.stream:
if (
request.response_format == "diarized_json"
and not self.model_cls.supports_diarized_transcription
):
return self.create_error_response(
"verbose_json format doesn't support streaming case"
f"Currently do not support diarized_json for {request.model}"
)
if (
request.response_format in {"verbose_json", "diarized_json"}
and request.stream
):
return self.create_error_response(
f"{request.response_format} format doesn't support streaming case"
)
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
@@ -634,7 +654,33 @@ class SpeechToTextBaseServing(GenerateBaseServing):
# rounded up as per openAI specs
"seconds": int(math.ceil(duration_s)),
}
if request.response_format != "verbose_json":
if request.response_format == "diarized_json":
diarized_segments = self.model_cls.parse_diarized_transcript(text)
if not diarized_segments:
return self.create_error_response(
"Model output did not contain a valid diarized transcript"
)
final_response = cast(
T,
TranscriptionResponseDiarized(
duration=duration_s,
text=separator.join(
segment.text for segment in diarized_segments
),
segments=[
{
"id": f"seg_{index}",
"start": segment.start,
"end": segment.end,
"text": segment.text,
"speaker": segment.speaker,
}
for index, segment in enumerate(diarized_segments)
],
usage=usage,
),
)
elif request.response_format != "verbose_json":
final_response = cast(
T, TranscriptionResponse(text=text, usage=usage)
)
@@ -27,7 +27,7 @@ from vllm.sampling_params import (
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
if TYPE_CHECKING:
import numpy as np
@@ -88,7 +88,7 @@ class TranscriptionRequest(OpenAIBaseModel):
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
response_format: TranscriptionResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
@@ -400,6 +400,27 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
"""Extracted words and their corresponding timestamps."""
class TranscriptionDiarizedSegment(OpenAIBaseModel):
"""A speaker-attributed transcription segment."""
type: Literal["transcript.text.segment"] = "transcript.text.segment"
id: str
start: float
end: float
text: str
speaker: str
class TranscriptionResponseDiarized(OpenAIBaseModel):
"""OpenAI-compatible diarized transcription response."""
task: Literal["transcribe"] = "transcribe"
duration: float
text: str
segments: list[TranscriptionDiarizedSegment]
usage: TranscriptionUsageAudio
TranscriptionResponseVariant: TypeAlias = (
TranscriptionResponse | TranscriptionResponseVerbose
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
)
@@ -18,6 +18,7 @@ from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionStreamResponse,
@@ -55,6 +56,7 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
) -> (
TranscriptionResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| AsyncGenerator[str, None]
| ErrorResponse
):
@@ -70,6 +72,8 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
response_class=(
TranscriptionResponseVerbose
if request.response_format == "verbose_json"
else TranscriptionResponseDiarized
if request.response_format == "diarized_json"
else TranscriptionResponse
),
stream_generator_method=self.transcription_stream_generator,