[Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543)

Signed-off-by: Lucas Christian <lucaschgf7@gmail.com>
This commit is contained in:
christian
2026-07-29 23:24:30 +00:00
committed by GitHub
parent 1cb3fe5844
commit 9d1aa4dda8
9 changed files with 309 additions and 10 deletions
@@ -8,4 +8,5 @@ import torch
## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
_LONG_INFO = torch.iinfo(torch.long)
@@ -43,6 +43,7 @@ from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterat
from ..transcription.protocol import (
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionSegment,
@@ -56,7 +57,9 @@ from ..translation.protocol import (
TranslationStreamResponse,
)
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponse: TypeAlias = (
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
)
SpeechToTextResponseVerbose: TypeAlias = (
TranscriptionResponseVerbose | TranslationResponseVerbose
)
@@ -70,6 +73,7 @@ ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| TranslationResponseVerbose
)
@@ -448,10 +452,15 @@ class SpeechToTextBaseServing(GenerateBaseServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
if request.response_format not in ["text", "json", "verbose_json"]:
if request.response_format not in [
"text",
"json",
"verbose_json",
"diarized_json",
]:
return self.create_error_response(
"Currently only support response_format: "
"`text`, `json` or `verbose_json`"
"`text`, `json`, `verbose_json` or `diarized_json`"
)
if (
@@ -462,9 +471,20 @@ class SpeechToTextBaseServing(GenerateBaseServing):
f"Currently do not support verbose_json for {request.model}"
)
if request.response_format == "verbose_json" and request.stream:
if (
request.response_format == "diarized_json"
and not self.model_cls.supports_diarized_transcription
):
return self.create_error_response(
"verbose_json format doesn't support streaming case"
f"Currently do not support diarized_json for {request.model}"
)
if (
request.response_format in {"verbose_json", "diarized_json"}
and request.stream
):
return self.create_error_response(
f"{request.response_format} format doesn't support streaming case"
)
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
@@ -634,7 +654,33 @@ class SpeechToTextBaseServing(GenerateBaseServing):
# rounded up as per openAI specs
"seconds": int(math.ceil(duration_s)),
}
if request.response_format != "verbose_json":
if request.response_format == "diarized_json":
diarized_segments = self.model_cls.parse_diarized_transcript(text)
if not diarized_segments:
return self.create_error_response(
"Model output did not contain a valid diarized transcript"
)
final_response = cast(
T,
TranscriptionResponseDiarized(
duration=duration_s,
text=separator.join(
segment.text for segment in diarized_segments
),
segments=[
{
"id": f"seg_{index}",
"start": segment.start,
"end": segment.end,
"text": segment.text,
"speaker": segment.speaker,
}
for index, segment in enumerate(diarized_segments)
],
usage=usage,
),
)
elif request.response_format != "verbose_json":
final_response = cast(
T, TranscriptionResponse(text=text, usage=usage)
)
@@ -27,7 +27,7 @@ from vllm.sampling_params import (
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
if TYPE_CHECKING:
import numpy as np
@@ -88,7 +88,7 @@ class TranscriptionRequest(OpenAIBaseModel):
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
response_format: TranscriptionResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
@@ -400,6 +400,27 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
"""Extracted words and their corresponding timestamps."""
class TranscriptionDiarizedSegment(OpenAIBaseModel):
"""A speaker-attributed transcription segment."""
type: Literal["transcript.text.segment"] = "transcript.text.segment"
id: str
start: float
end: float
text: str
speaker: str
class TranscriptionResponseDiarized(OpenAIBaseModel):
"""OpenAI-compatible diarized transcription response."""
task: Literal["transcribe"] = "transcribe"
duration: float
text: str
segments: list[TranscriptionDiarizedSegment]
usage: TranscriptionUsageAudio
TranscriptionResponseVariant: TypeAlias = (
TranscriptionResponse | TranscriptionResponseVerbose
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
)
@@ -18,6 +18,7 @@ from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionStreamResponse,
@@ -55,6 +56,7 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
) -> (
TranscriptionResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| AsyncGenerator[str, None]
| ErrorResponse
):
@@ -70,6 +72,8 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
response_class=(
TranscriptionResponseVerbose
if request.response_format == "verbose_json"
else TranscriptionResponseDiarized
if request.response_format == "diarized_json"
else TranscriptionResponse
),
stream_generator_method=self.transcription_stream_generator,
+23
View File
@@ -11,6 +11,7 @@ from collections.abc import (
Sequence,
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
@@ -71,6 +72,16 @@ The output embeddings must be one of the following formats:
"""
@dataclass(frozen=True, slots=True)
class DiarizedTranscriptionSegment:
"""A timestamped, speaker-attributed segment produced by an ASR model."""
start: float
end: float
speaker: str
text: str
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
@@ -1133,6 +1144,9 @@ class SupportsTranscription(Protocol):
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_diarized_transcription: ClassVar[bool] = False
"""Enables the ``diarized_json`` response format for the model."""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
@@ -1238,6 +1252,15 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse the model-specific diarized transcript format.
Only models that set ``supports_diarized_transcription`` must override
this method.
"""
raise NotImplementedError
@classmethod
def get_streaming_post_processor_cls(
cls,
@@ -14,6 +14,7 @@ import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import regex as re
import torch
from torch import nn
from transformers import BatchFeature
@@ -23,6 +24,7 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import (
DiarizedTranscriptionSegment,
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
@@ -35,6 +37,8 @@ from vllm.model_executor.models.utils import (
_merge_multimodal_embeddings,
init_vllm_registered_model,
maybe_prefix,
parse_diarized_speaker,
parse_diarized_timestamp,
)
from vllm.model_executor.models.whisper import (
WhisperEncoder,
@@ -77,6 +81,11 @@ DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = (
"并在段末标注结束时间戳,以清晰标明该段语音范围。"
)
_MOSS_DIARIZED_HEADER_RE = re.compile(
r"\[(?P<start>[0-9.]{1,32})\]\s*\[(?P<speaker>S[0-9]{1,15})\]"
)
_MOSS_DIARIZED_END_RE = re.compile(r"\[(?P<end>[0-9.]{1,32})\]\s*\Z")
class MossTranscribeDiarizeAudioInputs(TensorSchema):
"""
@@ -563,6 +572,7 @@ class MossTranscribeDiarizeForConditionalGeneration(
supports_transcription = True
supports_transcription_only = True
supports_segment_timestamp = False
supports_diarized_transcription = True
supported_languages = ISO639_1_SUPPORTED_LANGS
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -631,6 +641,46 @@ class MossTranscribeDiarizeForConditionalGeneration(
def post_process_output(cls, text: str) -> str:
return text.strip()
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse MOSS's canonical ``[start][Sxx]text[end]`` transcript."""
headers: list[tuple[re.Match[str], float, str]] = []
for match in _MOSS_DIARIZED_HEADER_RE.finditer(text):
start = parse_diarized_timestamp(match["start"])
speaker = parse_diarized_speaker(match["speaker"])
if start is not None and speaker is not None:
headers.append((match, start, speaker))
if not headers:
return []
segments: list[DiarizedTranscriptionSegment] = []
for index, (header, start, speaker) in enumerate(headers):
next_header_start = (
headers[index + 1][0].start() if index + 1 < len(headers) else len(text)
)
body = text[header.end() : next_header_start]
end_match = _MOSS_DIARIZED_END_RE.search(body)
if end_match is None:
return []
end = parse_diarized_timestamp(end_match["end"])
if end is None or end < start:
return []
segment_text = body[: end_match.start()].strip()
if segment_text:
segments.append(
DiarizedTranscriptionSegment(
start=start,
end=end,
speaker=speaker,
text=segment_text,
)
)
return segments
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.config = vllm_config.model_config.hf_config
+22
View File
@@ -1074,3 +1074,25 @@ def scatter_output_slices(
sliced = output[offset : offset + n_tok]
dest[idx] = sliced.clone() if clone else sliced
offset += n_tok
def parse_diarized_timestamp(marker: str) -> float | None:
if (
not marker
or not marker.isascii()
or marker.count(".") > 1
or not marker.replace(".", "").isdigit()
):
return None
return float(marker)
def parse_diarized_speaker(speaker: str) -> str | None:
if (
len(speaker) < 2
or speaker[0] != "S"
or not speaker[1:].isascii()
or not speaker[1:].isdigit()
):
return None
return speaker