From 9d1aa4dda85a58c74b23ff79aa4a20594ee6f359 Mon Sep 17 00:00:00 2001 From: christian Date: Wed, 29 Jul 2026 20:24:30 -0300 Subject: [PATCH] [Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543) Signed-off-by: Lucas Christian --- docs/serving/online_serving/speech_to_text.md | 26 ++++- .../test_moss_transcribe_diarize.py | 108 ++++++++++++++++++ .../speech_to_text/base/protocol.py | 1 + .../speech_to_text/base/serving.py | 58 +++++++++- .../speech_to_text/transcription/protocol.py | 27 ++++- .../speech_to_text/transcription/serving.py | 4 + vllm/model_executor/models/interfaces.py | 23 ++++ .../models/moss_transcribe_diarize.py | 50 ++++++++ vllm/model_executor/models/utils.py | 22 ++++ 9 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 tests/models/multimodal/processing/test_moss_transcribe_diarize.py diff --git a/docs/serving/online_serving/speech_to_text.md b/docs/serving/online_serving/speech_to_text.md index aa7636fb15..a956367cc3 100644 --- a/docs/serving/online_serving/speech_to_text.md +++ b/docs/serving/online_serving/speech_to_text.md @@ -64,7 +64,8 @@ The Transcriptions API supports uploading audio files in various formats includi - `model`: The model to use for transcription (required) - `language`: The language code (e.g., "en", "zh") (optional) - `prompt`: Optional text to guide the transcription style (optional) -- `response_format`: Format of the response ("json", "text") (optional) +- `response_format`: Format of the response ("json", "text", "verbose_json", or + "diarized_json") (optional) - `temperature`: Sampling temperature between 0 and 1 (optional) For the complete list of supported parameters including sampling parameters and vLLM extensions, see the [protocol definitions](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/speech_to_text/transcription/protocol.py). @@ -98,6 +99,29 @@ For `verbose_json` response format: ``` Currently “verbose_json” response format doesn’t support no_speech_prob. +For models with diarization support, `diarized_json` returns OpenAI-compatible +speaker segments. Currently, this is supported by +`OpenMOSS-Team/MOSS-Transcribe-Diarize`. + +```json +{ + "task": "transcribe", + "duration": 6.1, + "text": "Hello. Hi, how are you?", + "segments": [ + { + "type": "transcript.text.segment", + "id": "segment_0", + "start": 0.0, + "end": 2.8, + "text": "Hello.", + "speaker": "S01" + } + ], + "usage": {"type": "duration", "seconds": 7} +} +``` + ### Extra Parameters The following [sampling parameters](../../api/README.md#inference-parameters) are supported. diff --git a/tests/models/multimodal/processing/test_moss_transcribe_diarize.py b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py new file mode 100644 index 0000000000..954890d103 --- /dev/null +++ b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.model_executor.models.moss_transcribe_diarize import ( + MossTranscribeDiarizeForConditionalGeneration, +) + + +def test_parse_diarized_transcript_preserves_moss_segments(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]" + ) + + assert [ + (segment.start, segment.end, segment.speaker, segment.text) + for segment in segments + ] == [ + (0.48, 1.66, "S01", "Welcome"), + (12.26, 13.81, "S02", "Ready"), + ] + + +def test_parse_diarized_transcript_preserves_overlapping_segments(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01]First speaker[2][1][S02]Second speaker[3]" + ) + + assert [ + (segment.start, segment.end, segment.speaker, segment.text) + for segment in segments + ] == [ + (0.0, 2.0, "S01", "First speaker"), + (1.0, 3.0, "S02", "Second speaker"), + ] + + +def test_parse_diarized_transcript_preserves_numeric_text_markers(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01]The [2024] report is ready.[4]" + ) + + assert [segment.text for segment in segments] == ["The [2024] report is ready."] + + +def test_parse_diarized_transcript_ignores_whitespace_between_segments(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01]Hello[1]\n [2][S02]Hi[3]" + ) + + assert [(segment.start, segment.end, segment.text) for segment in segments] == [ + (0.0, 1.0, "Hello"), + (2.0, 3.0, "Hi"), + ] + + +def test_parse_diarized_transcript_ignores_noise_before_a_segment(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "noise [bad][0.1][S01]Hello[0.9]" + ) + + assert [(segment.start, segment.end, segment.text) for segment in segments] == [ + (0.1, 0.9, "Hello"), + ] + + +def test_parse_diarized_transcript_preserves_timestamps_before_the_end(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[2][S01]The earlier timestamp is [1] not the end[3]" + ) + + assert [segment.text for segment in segments] == [ + "The earlier timestamp is [1] not the end", + ] + + +def test_parse_diarized_transcript_skips_empty_segments(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01][1][2][S02]Complete[3]" + ) + + assert [(segment.speaker, segment.text) for segment in segments] == [ + ("S02", "Complete"), + ] + + +def test_parse_diarized_transcript_fails_closed_for_incomplete_output(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01]Complete[1][2][S02]Incomplete" + ) + + assert segments == [] + + +def test_parse_diarized_transcript_fails_closed_for_trailing_text(): + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + "[0][S01]Complete[1] trailing text" + ) + + assert segments == [] + + +def test_parse_diarized_transcript_preserves_overlong_timestamp_markers(): + marker = f"[{'1' * 33}]" + segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript( + f"[0][S01]Value {marker}[1]" + ) + + assert [segment.text for segment in segments] == [f"Value {marker}"] diff --git a/vllm/entrypoints/speech_to_text/base/protocol.py b/vllm/entrypoints/speech_to_text/base/protocol.py index e8cb61a414..70d28726e4 100644 --- a/vllm/entrypoints/speech_to_text/base/protocol.py +++ b/vllm/entrypoints/speech_to_text/base/protocol.py @@ -8,4 +8,5 @@ import torch ## Protocols for Audio AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"] +TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"] _LONG_INFO = torch.iinfo(torch.long) diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index 40f4cb25b1..e98816a132 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -43,6 +43,7 @@ from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterat from ..transcription.protocol import ( TranscriptionResponse, + TranscriptionResponseDiarized, TranscriptionResponseStreamChoice, TranscriptionResponseVerbose, TranscriptionSegment, @@ -56,7 +57,9 @@ from ..translation.protocol import ( TranslationStreamResponse, ) -SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse +SpeechToTextResponse: TypeAlias = ( + TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized +) SpeechToTextResponseVerbose: TypeAlias = ( TranscriptionResponseVerbose | TranslationResponseVerbose ) @@ -70,6 +73,7 @@ ResponseType: TypeAlias = ( TranscriptionResponse | TranslationResponse | TranscriptionResponseVerbose + | TranscriptionResponseDiarized | TranslationResponseVerbose ) @@ -448,10 +452,15 @@ class SpeechToTextBaseServing(GenerateBaseServing): if self.engine_client.errored: raise self.engine_client.dead_error - if request.response_format not in ["text", "json", "verbose_json"]: + if request.response_format not in [ + "text", + "json", + "verbose_json", + "diarized_json", + ]: return self.create_error_response( "Currently only support response_format: " - "`text`, `json` or `verbose_json`" + "`text`, `json`, `verbose_json` or `diarized_json`" ) if ( @@ -462,9 +471,20 @@ class SpeechToTextBaseServing(GenerateBaseServing): f"Currently do not support verbose_json for {request.model}" ) - if request.response_format == "verbose_json" and request.stream: + if ( + request.response_format == "diarized_json" + and not self.model_cls.supports_diarized_transcription + ): return self.create_error_response( - "verbose_json format doesn't support streaming case" + f"Currently do not support diarized_json for {request.model}" + ) + + if ( + request.response_format in {"verbose_json", "diarized_json"} + and request.stream + ): + return self.create_error_response( + f"{request.response_format} format doesn't support streaming case" ) request_id = f"{self.task_type}-{self._base_request_id(raw_request)}" @@ -634,7 +654,33 @@ class SpeechToTextBaseServing(GenerateBaseServing): # rounded up as per openAI specs "seconds": int(math.ceil(duration_s)), } - if request.response_format != "verbose_json": + if request.response_format == "diarized_json": + diarized_segments = self.model_cls.parse_diarized_transcript(text) + if not diarized_segments: + return self.create_error_response( + "Model output did not contain a valid diarized transcript" + ) + final_response = cast( + T, + TranscriptionResponseDiarized( + duration=duration_s, + text=separator.join( + segment.text for segment in diarized_segments + ), + segments=[ + { + "id": f"seg_{index}", + "start": segment.start, + "end": segment.end, + "text": segment.text, + "speaker": segment.speaker, + } + for index, segment in enumerate(diarized_segments) + ], + usage=usage, + ), + ) + elif request.response_format != "verbose_json": final_response = cast( T, TranscriptionResponse(text=text, usage=usage) ) diff --git a/vllm/entrypoints/speech_to_text/transcription/protocol.py b/vllm/entrypoints/speech_to_text/transcription/protocol.py index dba68abe7a..3220e15050 100644 --- a/vllm/entrypoints/speech_to_text/transcription/protocol.py +++ b/vllm/entrypoints/speech_to_text/transcription/protocol.py @@ -27,7 +27,7 @@ from vllm.sampling_params import ( ) from vllm.utils import random_uuid -from ..base.protocol import _LONG_INFO, AudioResponseFormat +from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat if TYPE_CHECKING: import numpy as np @@ -88,7 +88,7 @@ class TranscriptionRequest(OpenAIBaseModel): should match the audio language. """ - response_format: AudioResponseFormat = Field(default="json") + response_format: TranscriptionResponseFormat = Field(default="json") """ The format of the output, in one of these options: `json`, `text`, `srt`, `verbose_json`, or `vtt`. @@ -400,6 +400,27 @@ class TranscriptionResponseVerbose(OpenAIBaseModel): """Extracted words and their corresponding timestamps.""" +class TranscriptionDiarizedSegment(OpenAIBaseModel): + """A speaker-attributed transcription segment.""" + + type: Literal["transcript.text.segment"] = "transcript.text.segment" + id: str + start: float + end: float + text: str + speaker: str + + +class TranscriptionResponseDiarized(OpenAIBaseModel): + """OpenAI-compatible diarized transcription response.""" + + task: Literal["transcribe"] = "transcribe" + duration: float + text: str + segments: list[TranscriptionDiarizedSegment] + usage: TranscriptionUsageAudio + + TranscriptionResponseVariant: TypeAlias = ( - TranscriptionResponse | TranscriptionResponseVerbose + TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized ) diff --git a/vllm/entrypoints/speech_to_text/transcription/serving.py b/vllm/entrypoints/speech_to_text/transcription/serving.py index a59ae8aa31..51898841fa 100644 --- a/vllm/entrypoints/speech_to_text/transcription/serving.py +++ b/vllm/entrypoints/speech_to_text/transcription/serving.py @@ -18,6 +18,7 @@ from ..base.serving import SpeechToTextBaseServing from .protocol import ( TranscriptionRequest, TranscriptionResponse, + TranscriptionResponseDiarized, TranscriptionResponseStreamChoice, TranscriptionResponseVerbose, TranscriptionStreamResponse, @@ -55,6 +56,7 @@ class OpenAIServingTranscription(SpeechToTextBaseServing): ) -> ( TranscriptionResponse | TranscriptionResponseVerbose + | TranscriptionResponseDiarized | AsyncGenerator[str, None] | ErrorResponse ): @@ -70,6 +72,8 @@ class OpenAIServingTranscription(SpeechToTextBaseServing): response_class=( TranscriptionResponseVerbose if request.response_format == "verbose_json" + else TranscriptionResponseDiarized + if request.response_format == "diarized_json" else TranscriptionResponse ), stream_generator_method=self.transcription_stream_generator, diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 71229439f6..6556f6326e 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -11,6 +11,7 @@ from collections.abc import ( Sequence, ) from contextlib import ExitStack, contextmanager, nullcontext +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, @@ -71,6 +72,16 @@ The output embeddings must be one of the following formats: """ +@dataclass(frozen=True, slots=True) +class DiarizedTranscriptionSegment: + """A timestamped, speaker-attributed segment produced by an ASR model.""" + + start: float + end: float + speaker: str + text: str + + class StreamingTranscriptionPostProcessor: """Stateful streaming post-processor for transcription deltas.""" @@ -1133,6 +1144,9 @@ class SupportsTranscription(Protocol): Enables the segment timestamp option for supported models by setting this to `True`. """ + supports_diarized_transcription: ClassVar[bool] = False + """Enables the ``diarized_json`` response format for the model.""" + supports_explicit_language_detection: ClassVar[bool] = False """ Transcription models that require an explicit language detection step @@ -1238,6 +1252,15 @@ class SupportsTranscription(Protocol): """ return text + @classmethod + def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]: + """Parse the model-specific diarized transcript format. + + Only models that set ``supports_diarized_transcription`` must override + this method. + """ + raise NotImplementedError + @classmethod def get_streaming_post_processor_cls( cls, diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 789c789f0b..819881e0f2 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -14,6 +14,7 @@ import math from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Any, Literal, TypeAlias +import regex as re import torch from torch import nn from transformers import BatchFeature @@ -23,6 +24,7 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt from vllm.model_executor.models.interfaces import ( + DiarizedTranscriptionSegment, MultiModalEmbeddings, SupportsMultiModal, SupportsPP, @@ -35,6 +37,8 @@ from vllm.model_executor.models.utils import ( _merge_multimodal_embeddings, init_vllm_registered_model, maybe_prefix, + parse_diarized_speaker, + parse_diarized_timestamp, ) from vllm.model_executor.models.whisper import ( WhisperEncoder, @@ -77,6 +81,11 @@ DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = ( "并在段末标注结束时间戳,以清晰标明该段语音范围。" ) +_MOSS_DIARIZED_HEADER_RE = re.compile( + r"\[(?P[0-9.]{1,32})\]\s*\[(?PS[0-9]{1,15})\]" +) +_MOSS_DIARIZED_END_RE = re.compile(r"\[(?P[0-9.]{1,32})\]\s*\Z") + class MossTranscribeDiarizeAudioInputs(TensorSchema): """ @@ -563,6 +572,7 @@ class MossTranscribeDiarizeForConditionalGeneration( supports_transcription = True supports_transcription_only = True supports_segment_timestamp = False + supports_diarized_transcription = True supported_languages = ISO639_1_SUPPORTED_LANGS hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ @@ -631,6 +641,46 @@ class MossTranscribeDiarizeForConditionalGeneration( def post_process_output(cls, text: str) -> str: return text.strip() + @classmethod + def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]: + """Parse MOSS's canonical ``[start][Sxx]text[end]`` transcript.""" + headers: list[tuple[re.Match[str], float, str]] = [] + for match in _MOSS_DIARIZED_HEADER_RE.finditer(text): + start = parse_diarized_timestamp(match["start"]) + speaker = parse_diarized_speaker(match["speaker"]) + if start is not None and speaker is not None: + headers.append((match, start, speaker)) + + if not headers: + return [] + + segments: list[DiarizedTranscriptionSegment] = [] + for index, (header, start, speaker) in enumerate(headers): + next_header_start = ( + headers[index + 1][0].start() if index + 1 < len(headers) else len(text) + ) + body = text[header.end() : next_header_start] + end_match = _MOSS_DIARIZED_END_RE.search(body) + if end_match is None: + return [] + + end = parse_diarized_timestamp(end_match["end"]) + if end is None or end < start: + return [] + + segment_text = body[: end_match.start()].strip() + if segment_text: + segments.append( + DiarizedTranscriptionSegment( + start=start, + end=end, + speaker=speaker, + text=segment_text, + ) + ) + + return segments + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() self.config = vllm_config.model_config.hf_config diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index a6f39f6ebf..f92aa4f086 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -1074,3 +1074,25 @@ def scatter_output_slices( sliced = output[offset : offset + n_tok] dest[idx] = sliced.clone() if clone else sliced offset += n_tok + + +def parse_diarized_timestamp(marker: str) -> float | None: + if ( + not marker + or not marker.isascii() + or marker.count(".") > 1 + or not marker.replace(".", "").isdigit() + ): + return None + return float(marker) + + +def parse_diarized_speaker(speaker: str) -> str | None: + if ( + len(speaker) < 2 + or speaker[0] != "S" + or not speaker[1:].isascii() + or not speaker[1:].isdigit() + ): + return None + return speaker