[Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543)
Signed-off-by: Lucas Christian <lucaschgf7@gmail.com>
This commit is contained in:
@@ -64,7 +64,8 @@ The Transcriptions API supports uploading audio files in various formats includi
|
||||
- `model`: The model to use for transcription (required)
|
||||
- `language`: The language code (e.g., "en", "zh") (optional)
|
||||
- `prompt`: Optional text to guide the transcription style (optional)
|
||||
- `response_format`: Format of the response ("json", "text") (optional)
|
||||
- `response_format`: Format of the response ("json", "text", "verbose_json", or
|
||||
"diarized_json") (optional)
|
||||
- `temperature`: Sampling temperature between 0 and 1 (optional)
|
||||
|
||||
For the complete list of supported parameters including sampling parameters and vLLM extensions, see the [protocol definitions](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/speech_to_text/transcription/protocol.py).
|
||||
@@ -98,6 +99,29 @@ For `verbose_json` response format:
|
||||
```
|
||||
Currently “verbose_json” response format doesn’t support no_speech_prob.
|
||||
|
||||
For models with diarization support, `diarized_json` returns OpenAI-compatible
|
||||
speaker segments. Currently, this is supported by
|
||||
`OpenMOSS-Team/MOSS-Transcribe-Diarize`.
|
||||
|
||||
```json
|
||||
{
|
||||
"task": "transcribe",
|
||||
"duration": 6.1,
|
||||
"text": "Hello. Hi, how are you?",
|
||||
"segments": [
|
||||
{
|
||||
"type": "transcript.text.segment",
|
||||
"id": "segment_0",
|
||||
"start": 0.0,
|
||||
"end": 2.8,
|
||||
"text": "Hello.",
|
||||
"speaker": "S01"
|
||||
}
|
||||
],
|
||||
"usage": {"type": "duration", "seconds": 7}
|
||||
}
|
||||
```
|
||||
|
||||
### Extra Parameters
|
||||
|
||||
The following [sampling parameters](../../api/README.md#inference-parameters) are supported.
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.models.moss_transcribe_diarize import (
|
||||
MossTranscribeDiarizeForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_moss_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]"
|
||||
)
|
||||
|
||||
assert [
|
||||
(segment.start, segment.end, segment.speaker, segment.text)
|
||||
for segment in segments
|
||||
] == [
|
||||
(0.48, 1.66, "S01", "Welcome"),
|
||||
(12.26, 13.81, "S02", "Ready"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_overlapping_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]First speaker[2][1][S02]Second speaker[3]"
|
||||
)
|
||||
|
||||
assert [
|
||||
(segment.start, segment.end, segment.speaker, segment.text)
|
||||
for segment in segments
|
||||
] == [
|
||||
(0.0, 2.0, "S01", "First speaker"),
|
||||
(1.0, 3.0, "S02", "Second speaker"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_numeric_text_markers():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]The [2024] report is ready.[4]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == ["The [2024] report is ready."]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_ignores_whitespace_between_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Hello[1]\n [2][S02]Hi[3]"
|
||||
)
|
||||
|
||||
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
|
||||
(0.0, 1.0, "Hello"),
|
||||
(2.0, 3.0, "Hi"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_ignores_noise_before_a_segment():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"noise [bad][0.1][S01]Hello[0.9]"
|
||||
)
|
||||
|
||||
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
|
||||
(0.1, 0.9, "Hello"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_timestamps_before_the_end():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[2][S01]The earlier timestamp is [1] not the end[3]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == [
|
||||
"The earlier timestamp is [1] not the end",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_skips_empty_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01][1][2][S02]Complete[3]"
|
||||
)
|
||||
|
||||
assert [(segment.speaker, segment.text) for segment in segments] == [
|
||||
("S02", "Complete"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_fails_closed_for_incomplete_output():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Complete[1][2][S02]Incomplete"
|
||||
)
|
||||
|
||||
assert segments == []
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_fails_closed_for_trailing_text():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Complete[1] trailing text"
|
||||
)
|
||||
|
||||
assert segments == []
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_overlong_timestamp_markers():
|
||||
marker = f"[{'1' * 33}]"
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
f"[0][S01]Value {marker}[1]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == [f"Value {marker}"]
|
||||
@@ -8,4 +8,5 @@ import torch
|
||||
|
||||
## Protocols for Audio
|
||||
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
|
||||
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
|
||||
_LONG_INFO = torch.iinfo(torch.long)
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterat
|
||||
|
||||
from ..transcription.protocol import (
|
||||
TranscriptionResponse,
|
||||
TranscriptionResponseDiarized,
|
||||
TranscriptionResponseStreamChoice,
|
||||
TranscriptionResponseVerbose,
|
||||
TranscriptionSegment,
|
||||
@@ -56,7 +57,9 @@ from ..translation.protocol import (
|
||||
TranslationStreamResponse,
|
||||
)
|
||||
|
||||
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
|
||||
SpeechToTextResponse: TypeAlias = (
|
||||
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
|
||||
)
|
||||
SpeechToTextResponseVerbose: TypeAlias = (
|
||||
TranscriptionResponseVerbose | TranslationResponseVerbose
|
||||
)
|
||||
@@ -70,6 +73,7 @@ ResponseType: TypeAlias = (
|
||||
TranscriptionResponse
|
||||
| TranslationResponse
|
||||
| TranscriptionResponseVerbose
|
||||
| TranscriptionResponseDiarized
|
||||
| TranslationResponseVerbose
|
||||
)
|
||||
|
||||
@@ -448,10 +452,15 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
if request.response_format not in ["text", "json", "verbose_json"]:
|
||||
if request.response_format not in [
|
||||
"text",
|
||||
"json",
|
||||
"verbose_json",
|
||||
"diarized_json",
|
||||
]:
|
||||
return self.create_error_response(
|
||||
"Currently only support response_format: "
|
||||
"`text`, `json` or `verbose_json`"
|
||||
"`text`, `json`, `verbose_json` or `diarized_json`"
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -462,9 +471,20 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
f"Currently do not support verbose_json for {request.model}"
|
||||
)
|
||||
|
||||
if request.response_format == "verbose_json" and request.stream:
|
||||
if (
|
||||
request.response_format == "diarized_json"
|
||||
and not self.model_cls.supports_diarized_transcription
|
||||
):
|
||||
return self.create_error_response(
|
||||
"verbose_json format doesn't support streaming case"
|
||||
f"Currently do not support diarized_json for {request.model}"
|
||||
)
|
||||
|
||||
if (
|
||||
request.response_format in {"verbose_json", "diarized_json"}
|
||||
and request.stream
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"{request.response_format} format doesn't support streaming case"
|
||||
)
|
||||
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
|
||||
|
||||
@@ -634,7 +654,33 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
# rounded up as per openAI specs
|
||||
"seconds": int(math.ceil(duration_s)),
|
||||
}
|
||||
if request.response_format != "verbose_json":
|
||||
if request.response_format == "diarized_json":
|
||||
diarized_segments = self.model_cls.parse_diarized_transcript(text)
|
||||
if not diarized_segments:
|
||||
return self.create_error_response(
|
||||
"Model output did not contain a valid diarized transcript"
|
||||
)
|
||||
final_response = cast(
|
||||
T,
|
||||
TranscriptionResponseDiarized(
|
||||
duration=duration_s,
|
||||
text=separator.join(
|
||||
segment.text for segment in diarized_segments
|
||||
),
|
||||
segments=[
|
||||
{
|
||||
"id": f"seg_{index}",
|
||||
"start": segment.start,
|
||||
"end": segment.end,
|
||||
"text": segment.text,
|
||||
"speaker": segment.speaker,
|
||||
}
|
||||
for index, segment in enumerate(diarized_segments)
|
||||
],
|
||||
usage=usage,
|
||||
),
|
||||
)
|
||||
elif request.response_format != "verbose_json":
|
||||
final_response = cast(
|
||||
T, TranscriptionResponse(text=text, usage=usage)
|
||||
)
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.sampling_params import (
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import _LONG_INFO, AudioResponseFormat
|
||||
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
@@ -88,7 +88,7 @@ class TranscriptionRequest(OpenAIBaseModel):
|
||||
should match the audio language.
|
||||
"""
|
||||
|
||||
response_format: AudioResponseFormat = Field(default="json")
|
||||
response_format: TranscriptionResponseFormat = Field(default="json")
|
||||
"""
|
||||
The format of the output, in one of these options: `json`, `text`, `srt`,
|
||||
`verbose_json`, or `vtt`.
|
||||
@@ -400,6 +400,27 @@ class TranscriptionResponseVerbose(OpenAIBaseModel):
|
||||
"""Extracted words and their corresponding timestamps."""
|
||||
|
||||
|
||||
class TranscriptionDiarizedSegment(OpenAIBaseModel):
|
||||
"""A speaker-attributed transcription segment."""
|
||||
|
||||
type: Literal["transcript.text.segment"] = "transcript.text.segment"
|
||||
id: str
|
||||
start: float
|
||||
end: float
|
||||
text: str
|
||||
speaker: str
|
||||
|
||||
|
||||
class TranscriptionResponseDiarized(OpenAIBaseModel):
|
||||
"""OpenAI-compatible diarized transcription response."""
|
||||
|
||||
task: Literal["transcribe"] = "transcribe"
|
||||
duration: float
|
||||
text: str
|
||||
segments: list[TranscriptionDiarizedSegment]
|
||||
usage: TranscriptionUsageAudio
|
||||
|
||||
|
||||
TranscriptionResponseVariant: TypeAlias = (
|
||||
TranscriptionResponse | TranscriptionResponseVerbose
|
||||
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranscriptionResponseDiarized,
|
||||
TranscriptionResponseStreamChoice,
|
||||
TranscriptionResponseVerbose,
|
||||
TranscriptionStreamResponse,
|
||||
@@ -55,6 +56,7 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
| TranscriptionResponseVerbose
|
||||
| TranscriptionResponseDiarized
|
||||
| AsyncGenerator[str, None]
|
||||
| ErrorResponse
|
||||
):
|
||||
@@ -70,6 +72,8 @@ class OpenAIServingTranscription(SpeechToTextBaseServing):
|
||||
response_class=(
|
||||
TranscriptionResponseVerbose
|
||||
if request.response_format == "verbose_json"
|
||||
else TranscriptionResponseDiarized
|
||||
if request.response_format == "diarized_json"
|
||||
else TranscriptionResponse
|
||||
),
|
||||
stream_generator_method=self.transcription_stream_generator,
|
||||
|
||||
@@ -11,6 +11,7 @@ from collections.abc import (
|
||||
Sequence,
|
||||
)
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
@@ -71,6 +72,16 @@ The output embeddings must be one of the following formats:
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DiarizedTranscriptionSegment:
|
||||
"""A timestamped, speaker-attributed segment produced by an ASR model."""
|
||||
|
||||
start: float
|
||||
end: float
|
||||
speaker: str
|
||||
text: str
|
||||
|
||||
|
||||
class StreamingTranscriptionPostProcessor:
|
||||
"""Stateful streaming post-processor for transcription deltas."""
|
||||
|
||||
@@ -1133,6 +1144,9 @@ class SupportsTranscription(Protocol):
|
||||
Enables the segment timestamp option for supported models by setting this to `True`.
|
||||
"""
|
||||
|
||||
supports_diarized_transcription: ClassVar[bool] = False
|
||||
"""Enables the ``diarized_json`` response format for the model."""
|
||||
|
||||
supports_explicit_language_detection: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models that require an explicit language detection step
|
||||
@@ -1238,6 +1252,15 @@ class SupportsTranscription(Protocol):
|
||||
"""
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
|
||||
"""Parse the model-specific diarized transcript format.
|
||||
|
||||
Only models that set ``supports_diarized_transcription`` must override
|
||||
this method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
|
||||
@@ -14,6 +14,7 @@ import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import BatchFeature
|
||||
@@ -23,6 +24,7 @@ from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
DiarizedTranscriptionSegment,
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -35,6 +37,8 @@ from vllm.model_executor.models.utils import (
|
||||
_merge_multimodal_embeddings,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
parse_diarized_speaker,
|
||||
parse_diarized_timestamp,
|
||||
)
|
||||
from vllm.model_executor.models.whisper import (
|
||||
WhisperEncoder,
|
||||
@@ -77,6 +81,11 @@ DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = (
|
||||
"并在段末标注结束时间戳,以清晰标明该段语音范围。"
|
||||
)
|
||||
|
||||
_MOSS_DIARIZED_HEADER_RE = re.compile(
|
||||
r"\[(?P<start>[0-9.]{1,32})\]\s*\[(?P<speaker>S[0-9]{1,15})\]"
|
||||
)
|
||||
_MOSS_DIARIZED_END_RE = re.compile(r"\[(?P<end>[0-9.]{1,32})\]\s*\Z")
|
||||
|
||||
|
||||
class MossTranscribeDiarizeAudioInputs(TensorSchema):
|
||||
"""
|
||||
@@ -563,6 +572,7 @@ class MossTranscribeDiarizeForConditionalGeneration(
|
||||
supports_transcription = True
|
||||
supports_transcription_only = True
|
||||
supports_segment_timestamp = False
|
||||
supports_diarized_transcription = True
|
||||
supported_languages = ISO639_1_SUPPORTED_LANGS
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -631,6 +641,46 @@ class MossTranscribeDiarizeForConditionalGeneration(
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
return text.strip()
|
||||
|
||||
@classmethod
|
||||
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
|
||||
"""Parse MOSS's canonical ``[start][Sxx]text[end]`` transcript."""
|
||||
headers: list[tuple[re.Match[str], float, str]] = []
|
||||
for match in _MOSS_DIARIZED_HEADER_RE.finditer(text):
|
||||
start = parse_diarized_timestamp(match["start"])
|
||||
speaker = parse_diarized_speaker(match["speaker"])
|
||||
if start is not None and speaker is not None:
|
||||
headers.append((match, start, speaker))
|
||||
|
||||
if not headers:
|
||||
return []
|
||||
|
||||
segments: list[DiarizedTranscriptionSegment] = []
|
||||
for index, (header, start, speaker) in enumerate(headers):
|
||||
next_header_start = (
|
||||
headers[index + 1][0].start() if index + 1 < len(headers) else len(text)
|
||||
)
|
||||
body = text[header.end() : next_header_start]
|
||||
end_match = _MOSS_DIARIZED_END_RE.search(body)
|
||||
if end_match is None:
|
||||
return []
|
||||
|
||||
end = parse_diarized_timestamp(end_match["end"])
|
||||
if end is None or end < start:
|
||||
return []
|
||||
|
||||
segment_text = body[: end_match.start()].strip()
|
||||
if segment_text:
|
||||
segments.append(
|
||||
DiarizedTranscriptionSegment(
|
||||
start=start,
|
||||
end=end,
|
||||
speaker=speaker,
|
||||
text=segment_text,
|
||||
)
|
||||
)
|
||||
|
||||
return segments
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -1074,3 +1074,25 @@ def scatter_output_slices(
|
||||
sliced = output[offset : offset + n_tok]
|
||||
dest[idx] = sliced.clone() if clone else sliced
|
||||
offset += n_tok
|
||||
|
||||
|
||||
def parse_diarized_timestamp(marker: str) -> float | None:
|
||||
if (
|
||||
not marker
|
||||
or not marker.isascii()
|
||||
or marker.count(".") > 1
|
||||
or not marker.replace(".", "").isdigit()
|
||||
):
|
||||
return None
|
||||
return float(marker)
|
||||
|
||||
|
||||
def parse_diarized_speaker(speaker: str) -> str | None:
|
||||
if (
|
||||
len(speaker) < 2
|
||||
or speaker[0] != "S"
|
||||
or not speaker[1:].isascii()
|
||||
or not speaker[1:].isdigit()
|
||||
):
|
||||
return None
|
||||
return speaker
|
||||
|
||||
Reference in New Issue
Block a user