[Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543)

Signed-off-by: Lucas Christian <lucaschgf7@gmail.com>
This commit is contained in:
christian
2026-07-29 23:24:30 +00:00
committed by GitHub
parent 1cb3fe5844
commit 9d1aa4dda8
9 changed files with 309 additions and 10 deletions
+25 -1
View File
@@ -64,7 +64,8 @@ The Transcriptions API supports uploading audio files in various formats includi
- `model`: The model to use for transcription (required)
- `language`: The language code (e.g., "en", "zh") (optional)
- `prompt`: Optional text to guide the transcription style (optional)
- `response_format`: Format of the response ("json", "text") (optional)
- `response_format`: Format of the response ("json", "text", "verbose_json", or
"diarized_json") (optional)
- `temperature`: Sampling temperature between 0 and 1 (optional)
For the complete list of supported parameters including sampling parameters and vLLM extensions, see the [protocol definitions](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/speech_to_text/transcription/protocol.py).
@@ -98,6 +99,29 @@ For `verbose_json` response format:
```
Currently “verbose_json” response format doesn’t support no_speech_prob.
For models with diarization support, `diarized_json` returns OpenAI-compatible
speaker segments. Currently, this is supported by
`OpenMOSS-Team/MOSS-Transcribe-Diarize`.
```json
{
"task": "transcribe",
"duration": 6.1,
"text": "Hello. Hi, how are you?",
"segments": [
{
"type": "transcript.text.segment",
"id": "segment_0",
"start": 0.0,
"end": 2.8,
"text": "Hello.",
"speaker": "S01"
}
],
"usage": {"type": "duration", "seconds": 7}
}
```
### Extra Parameters
The following [sampling parameters](../../api/README.md#inference-parameters) are supported.
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.models.moss_transcribe_diarize import (
MossTranscribeDiarizeForConditionalGeneration,
)
def test_parse_diarized_transcript_preserves_moss_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]"
)
assert [
(segment.start, segment.end, segment.speaker, segment.text)
for segment in segments
] == [
(0.48, 1.66, "S01", "Welcome"),
(12.26, 13.81, "S02", "Ready"),
]
def test_parse_diarized_transcript_preserves_overlapping_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]First speaker[2][1][S02]Second speaker[3]"
)
assert [
(segment.start, segment.end, segment.speaker, segment.text)
for segment in segments
] == [
(0.0, 2.0, "S01", "First speaker"),
(1.0, 3.0, "S02", "Second speaker"),
]
def test_parse_diarized_transcript_preserves_numeric_text_markers():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]The [2024] report is ready.[4]"
)
assert [segment.text for segment in segments] == ["The [2024] report is ready."]
def test_parse_diarized_transcript_ignores_whitespace_between_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Hello[1]\n [2][S02]Hi[3]"
)
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
(0.0, 1.0, "Hello"),
(2.0, 3.0, "Hi"),
]
def test_parse_diarized_transcript_ignores_noise_before_a_segment():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"noise [bad][0.1][S01]Hello[0.9]"
)
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
(0.1, 0.9, "Hello"),
]
def test_parse_diarized_transcript_preserves_timestamps_before_the_end():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[2][S01]The earlier timestamp is [1] not the end[3]"
)
assert [segment.text for segment in segments] == [
"The earlier timestamp is [1] not the end",
]
def test_parse_diarized_transcript_skips_empty_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01][1][2][S02]Complete[3]"
)
assert [(segment.speaker, segment.text) for segment in segments] == [
("S02", "Complete"),
]
def test_parse_diarized_transcript_fails_closed_for_incomplete_output():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Complete[1][2][S02]Incomplete"
)
assert segments == []
def test_parse_diarized_transcript_fails_closed_for_trailing_text():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Complete[1] trailing text"
)
assert segments == []
def test_parse_diarized_transcript_preserves_overlong_timestamp_markers():
marker = f"[{'1' * 33}]"
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
f"[0][S01]Value {marker}[1]"
)
assert [segment.text for segment in segments] == [f"Value {marker}"]
@@ -8,4 +8,5 @@ import torch
## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
_LONG_INFO = torch.iinfo(torch.long)
@@ -43,6 +43,7 @@ from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterat
from ..transcription.protocol import (
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionSegment,
@@ -56,7 +57,9 @@ from ..translation.protocol import (
TranslationStreamResponse,
)
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponse: TypeAlias = (
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
)
SpeechToTextResponseVerbose: TypeAlias = (
TranscriptionResponseVerbose | TranslationResponseVerbose
)
@@ -70,6 +73,7 @@ ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| TranslationResponseVerbose
)
@@ -448,10 +452,15 @@ class SpeechToTextBaseServing(GenerateBaseServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
if request.response_format not in ["text", "json", "verbose_json"]:
if request.response_format not in [
"text",
"json",
"verbose_json",
"diarized_json",
]:
return self.create_error_response(
"Currently only support response_format: "
"`text`, `json` or `verbose_json`"
"`text`, `json`, `verbose_json` or `diarized_json`"
)
if (
@@ -462,9 +471,20 @@ class SpeechToTextBaseServing(GenerateBaseServing):
f"Currently do not support verbose_json for {request.model}"
)
if request.response_format == "verbose_json" and request.stream:
if (
request.response_format == "diarized_json"
and not self.model_cls.supports_diarized_transcription
):
return self.create_error_response(
"verbose_json format doesn't support streaming case"
f"Currently do not support diarized_json for {request.model}"
)
if (
request.response_format in {"verbose_json", "diarized_json"}
and request.stream
):
return self.create_error_response(
f"{request.response_format} format doesn't support streaming case"
)
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
@@ -634,7 +654,33 @@ class SpeechToTextBaseServing(GenerateBaseServing):
# rounded up as per openAI specs
"seconds": int(math.ceil(duration_s)),
}
if request.response_format != "verbose_json":
if request.response_format == "diarized_json":
diarized_segments = self.model_cls.parse_diarized_transcript(text)
if not diarized_segments:
return self.create_error_response(
"Model output did not contain a valid diarized transcript"
)
final_response = cast(
T,
TranscriptionResponseDiarized(
duration=duration_s,
text=separator.join(
segment.text for segment in diarized_segments
),
segments=[
{
"id": f"seg_{index}",
"start": segment.start,
"end": segment.end,
"text": segment.text,
"speaker": segment.speaker,
}
for index, segment in enumerate(diarized_segments)
],
usage=usage,
),
)
elif request.response_format != "verbose_json":
final_response = cast(
T, TranscriptionResponse(text=text, usage=usage)
)
@@ -27,7 +27,7 @@ from vllm.sampling_params import (
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
if TYPE_CHECKING:
import numpy as np
@@ -88,7 +88,7 @@ class TranscriptionRequest(OpenAIBaseModel):
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
response_format: TranscriptionResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
@@ -400,6 +400,27 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
"""Extracted words and their corresponding timestamps."""
class TranscriptionDiarizedSegment(OpenAIBaseModel):
"""A speaker-attributed transcription segment."""
type: Literal["transcript.text.segment"] = "transcript.text.segment"
id: str
start: float
end: float
text: str
speaker: str
class TranscriptionResponseDiarized(OpenAIBaseModel):
"""OpenAI-compatible diarized transcription response."""
task: Literal["transcribe"] = "transcribe"
duration: float
text: str
segments: list[TranscriptionDiarizedSegment]
usage: TranscriptionUsageAudio
TranscriptionResponseVariant: TypeAlias = (
TranscriptionResponse | TranscriptionResponseVerbose
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
)
@@ -18,6 +18,7 @@ from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionStreamResponse,
@@ -55,6 +56,7 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
) -> (
TranscriptionResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| AsyncGenerator[str, None]
| ErrorResponse
):
@@ -70,6 +72,8 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
response_class=(
TranscriptionResponseVerbose
if request.response_format == "verbose_json"
else TranscriptionResponseDiarized
if request.response_format == "diarized_json"
else TranscriptionResponse
),
stream_generator_method=self.transcription_stream_generator,
+23
View File
@@ -11,6 +11,7 @@ from collections.abc import (
Sequence,
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
@@ -71,6 +72,16 @@ The output embeddings must be one of the following formats:
"""
@dataclass(frozen=True, slots=True)
class DiarizedTranscriptionSegment:
"""A timestamped, speaker-attributed segment produced by an ASR model."""
start: float
end: float
speaker: str
text: str
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
@@ -1133,6 +1144,9 @@ class SupportsTranscription(Protocol):
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_diarized_transcription: ClassVar[bool] = False
"""Enables the ``diarized_json`` response format for the model."""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
@@ -1238,6 +1252,15 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse the model-specific diarized transcript format.
Only models that set ``supports_diarized_transcription`` must override
this method.
"""
raise NotImplementedError
@classmethod
def get_streaming_post_processor_cls(
cls,
@@ -14,6 +14,7 @@ import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import regex as re
import torch
from torch import nn
from transformers import BatchFeature
@@ -23,6 +24,7 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import (
DiarizedTranscriptionSegment,
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
@@ -35,6 +37,8 @@ from vllm.model_executor.models.utils import (
_merge_multimodal_embeddings,
init_vllm_registered_model,
maybe_prefix,
parse_diarized_speaker,
parse_diarized_timestamp,
)
from vllm.model_executor.models.whisper import (
WhisperEncoder,
@@ -77,6 +81,11 @@ DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = (
"并在段末标注结束时间戳,以清晰标明该段语音范围。"
)
_MOSS_DIARIZED_HEADER_RE = re.compile(
r"\[(?P<start>[0-9.]{1,32})\]\s*\[(?P<speaker>S[0-9]{1,15})\]"
)
_MOSS_DIARIZED_END_RE = re.compile(r"\[(?P<end>[0-9.]{1,32})\]\s*\Z")
class MossTranscribeDiarizeAudioInputs(TensorSchema):
"""
@@ -563,6 +572,7 @@ class MossTranscribeDiarizeForConditionalGeneration(
supports_transcription = True
supports_transcription_only = True
supports_segment_timestamp = False
supports_diarized_transcription = True
supported_languages = ISO639_1_SUPPORTED_LANGS
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -631,6 +641,46 @@ class MossTranscribeDiarizeForConditionalGeneration(
def post_process_output(cls, text: str) -> str:
return text.strip()
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse MOSS's canonical ``[start][Sxx]text[end]`` transcript."""
headers: list[tuple[re.Match[str], float, str]] = []
for match in _MOSS_DIARIZED_HEADER_RE.finditer(text):
start = parse_diarized_timestamp(match["start"])
speaker = parse_diarized_speaker(match["speaker"])
if start is not None and speaker is not None:
headers.append((match, start, speaker))
if not headers:
return []
segments: list[DiarizedTranscriptionSegment] = []
for index, (header, start, speaker) in enumerate(headers):
next_header_start = (
headers[index + 1][0].start() if index + 1 < len(headers) else len(text)
)
body = text[header.end() : next_header_start]
end_match = _MOSS_DIARIZED_END_RE.search(body)
if end_match is None:
return []
end = parse_diarized_timestamp(end_match["end"])
if end is None or end < start:
return []
segment_text = body[: end_match.start()].strip()
if segment_text:
segments.append(
DiarizedTranscriptionSegment(
start=start,
end=end,
speaker=speaker,
text=segment_text,
)
)
return segments
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.config = vllm_config.model_config.hf_config
+22
View File
@@ -1074,3 +1074,25 @@ def scatter_output_slices(
sliced = output[offset : offset + n_tok]
dest[idx] = sliced.clone() if clone else sliced
offset += n_tok
def parse_diarized_timestamp(marker: str) -> float | None:
if (
not marker
or not marker.isascii()
or marker.count(".") > 1
or not marker.replace(".", "").isdigit()
):
return None
return float(marker)
def parse_diarized_speaker(speaker: str) -> str | None:
if (
len(speaker) < 2
or speaker[0] != "S"
or not speaker[1:].isascii()
or not speaker[1:].isdigit()
):
return None
return speaker