[Frontend] Add diarized_json support for MOSS-Transcribe-Diarize (#48543)

Signed-off-by: Lucas Christian <lucaschgf7@gmail.com>
This commit is contained in:
christian
2026-07-29 23:24:30 +00:00
committed by GitHub
parent 1cb3fe5844
commit 9d1aa4dda8
9 changed files with 309 additions and 10 deletions
+23
View File
@@ -11,6 +11,7 @@ from collections.abc import (
Sequence,
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
@@ -71,6 +72,16 @@ The output embeddings must be one of the following formats:
"""
@dataclass(frozen=True, slots=True)
class DiarizedTranscriptionSegment:
"""A timestamped, speaker-attributed segment produced by an ASR model."""
start: float
end: float
speaker: str
text: str
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
@@ -1133,6 +1144,9 @@ class SupportsTranscription(Protocol):
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_diarized_transcription: ClassVar[bool] = False
"""Enables the ``diarized_json`` response format for the model."""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
@@ -1238,6 +1252,15 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse the model-specific diarized transcript format.
Only models that set ``supports_diarized_transcription`` must override
this method.
"""
raise NotImplementedError
@classmethod
def get_streaming_post_processor_cls(
cls,
@@ -14,6 +14,7 @@ import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import regex as re
import torch
from torch import nn
from transformers import BatchFeature
@@ -23,6 +24,7 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import (
DiarizedTranscriptionSegment,
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
@@ -35,6 +37,8 @@ from vllm.model_executor.models.utils import (
_merge_multimodal_embeddings,
init_vllm_registered_model,
maybe_prefix,
parse_diarized_speaker,
parse_diarized_timestamp,
)
from vllm.model_executor.models.whisper import (
WhisperEncoder,
@@ -77,6 +81,11 @@ DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = (
"并在段末标注结束时间戳,以清晰标明该段语音范围。"
)
_MOSS_DIARIZED_HEADER_RE = re.compile(
r"\[(?P<start>[0-9.]{1,32})\]\s*\[(?P<speaker>S[0-9]{1,15})\]"
)
_MOSS_DIARIZED_END_RE = re.compile(r"\[(?P<end>[0-9.]{1,32})\]\s*\Z")
class MossTranscribeDiarizeAudioInputs(TensorSchema):
"""
@@ -563,6 +572,7 @@ class MossTranscribeDiarizeForConditionalGeneration(
supports_transcription = True
supports_transcription_only = True
supports_segment_timestamp = False
supports_diarized_transcription = True
supported_languages = ISO639_1_SUPPORTED_LANGS
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -631,6 +641,46 @@ class MossTranscribeDiarizeForConditionalGeneration(
def post_process_output(cls, text: str) -> str:
return text.strip()
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse MOSS's canonical ``[start][Sxx]text[end]`` transcript."""
headers: list[tuple[re.Match[str], float, str]] = []
for match in _MOSS_DIARIZED_HEADER_RE.finditer(text):
start = parse_diarized_timestamp(match["start"])
speaker = parse_diarized_speaker(match["speaker"])
if start is not None and speaker is not None:
headers.append((match, start, speaker))
if not headers:
return []
segments: list[DiarizedTranscriptionSegment] = []
for index, (header, start, speaker) in enumerate(headers):
next_header_start = (
headers[index + 1][0].start() if index + 1 < len(headers) else len(text)
)
body = text[header.end() : next_header_start]
end_match = _MOSS_DIARIZED_END_RE.search(body)
if end_match is None:
return []
end = parse_diarized_timestamp(end_match["end"])
if end is None or end < start:
return []
segment_text = body[: end_match.start()].strip()
if segment_text:
segments.append(
DiarizedTranscriptionSegment(
start=start,
end=end,
speaker=speaker,
text=segment_text,
)
)
return segments
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.config = vllm_config.model_config.hf_config
+22
View File
@@ -1074,3 +1074,25 @@ def scatter_output_slices(
sliced = output[offset : offset + n_tok]
dest[idx] = sliced.clone() if clone else sliced
offset += n_tok
def parse_diarized_timestamp(marker: str) -> float | None:
if (
not marker
or not marker.isascii()
or marker.count(".") > 1
or not marker.replace(".", "").isdigit()
):
return None
return float(marker)
def parse_diarized_speaker(speaker: str) -> str | None:
if (
len(speaker) < 2
or speaker[0] != "S"
or not speaker[1:].isascii()
or not speaker[1:].isdigit()
):
return None
return speaker