diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index daca5e0cd8..a191bb2d3c 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -569,6 +569,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `MolmoForCausalLM` | Molmo | T + I+ | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ |
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`^, `allenai/MolmoWeb-8B`^ | ✅︎ | ✅︎ |
| `MossAudioModel` | MOSS-Audio | T + A+ | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |
+| `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | T + A | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ |
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
| `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
@@ -671,6 +672,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
+| `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ |
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ |
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 00085f44a6..91d5f63ab7 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1127,6 +1127,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
trust_remote_code=True,
),
+ "MossTranscribeDiarizeForConditionalGeneration": _HfExamplesInfo(
+ "OpenMOSS-Team/MOSS-Transcribe-Diarize",
+ trust_remote_code=True,
+ is_available_online=False,
+ ),
"HfMoondream": _HfExamplesInfo(
"moondream/moondream3-preview",
tokenizer="moondream/starmie-v1",
diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py
new file mode 100644
index 0000000000..5236f5c61b
--- /dev/null
+++ b/vllm/model_executor/models/moss_transcribe_diarize.py
@@ -0,0 +1,801 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Inference-only MOSS-Transcribe-Diarize ASR model.
+
+The checkpoint layout is:
+
+* ``model.whisper_encoder.*``: Whisper-medium encoder weights.
+* ``model.vq_adaptor.*``: 4x time-merge projector.
+* ``model.language_model.*``: Qwen3-0.6B decoder weights.
+"""
+
+import math
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Annotated, Any, Literal, TypeAlias
+
+import torch
+from torch import nn
+from transformers import BatchFeature
+
+from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
+from vllm.config.multimodal import BaseDummyOptions
+from vllm.config.speech_to_text import SpeechToTextParams
+from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
+from vllm.model_executor.models.interfaces import (
+ MultiModalEmbeddings,
+ SupportsMultiModal,
+ SupportsPP,
+ SupportsTranscription,
+ _require_is_multimodal,
+)
+from vllm.model_executor.models.utils import (
+ AutoWeightsLoader,
+ WeightsMapper,
+ _merge_multimodal_embeddings,
+ init_vllm_registered_model,
+ maybe_prefix,
+)
+from vllm.model_executor.models.whisper import (
+ WhisperEncoder,
+ _create_fake_bias_for_k_proj,
+)
+from vllm.model_executor.models.whisper_utils import ISO639_1_SUPPORTED_LANGS
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+ AudioItem,
+ MultiModalFieldConfig,
+ MultiModalKwargsItems,
+)
+from vllm.multimodal.parse import (
+ DictEmbeddingItems,
+ ModalityDataItems,
+ MultiModalDataItems,
+ MultiModalDataParser,
+)
+from vllm.multimodal.processing import (
+ BaseDummyInputsBuilder,
+ BaseMultiModalProcessor,
+ BaseProcessingInfo,
+ PromptReplacement,
+ PromptUpdate,
+ PromptUpdateDetails,
+)
+from vllm.multimodal.processing.processor import ProcessorInputs
+from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.processor import cached_processor_from_config
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+WHISPER_ENCODER_STRIDE = 2
+
+AUDIO_PLACEHOLDER = "<|audio_start|><|audio_pad|><|audio_end|>"
+
+DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT = (
+ "请将音频转写为文本,每一段需以起始时间戳和说话人编号"
+ "([S01]、[S02]、[S03]…)开头,正文为对应的语音内容,"
+ "并在段末标注结束时间戳,以清晰标明该段语音范围。"
+)
+
+
+class MossTranscribeDiarizeAudioInputs(TensorSchema):
+ """
+ Dimensions:
+ - c: Audio chunks
+ - m: Mel bins
+ - f: Mel frames
+ - n: Number of audio items
+ """
+
+ type: Literal["audio_features"] = "audio_features"
+
+ input_features: Annotated[
+ torch.Tensor | None,
+ TensorShape("c", "m", "f"),
+ ]
+ audio_feature_lengths: Annotated[
+ torch.Tensor | None,
+ TensorShape("c"),
+ ]
+ audio_chunk_counts: Annotated[
+ torch.Tensor | None,
+ TensorShape("n"),
+ ]
+
+
+class MossTranscribeDiarizeEmbeddingInputs(TensorSchema):
+ """
+ Dimensions:
+ - n: Number of audio items
+ - t: Number of audio tokens
+ - h: Hidden size
+ """
+
+ type: Literal["audio_embeds"] = "audio_embeds"
+
+ audio_embeds: Annotated[
+ list[torch.Tensor],
+ TensorShape("n", "t", "h", dynamic_dims={"t"}),
+ ]
+
+
+MossTranscribeDiarizeInputs: TypeAlias = (
+ MossTranscribeDiarizeAudioInputs | MossTranscribeDiarizeEmbeddingInputs
+)
+
+
+def _compute_total_audio_tokens(
+ num_samples: int,
+ feature_extractor: Any,
+ audio_merge_size: int,
+) -> int:
+ if num_samples <= 0:
+ return 0
+
+ n_samples = int(feature_extractor.n_samples)
+ stride = (
+ int(feature_extractor.hop_length)
+ * WHISPER_ENCODER_STRIDE
+ * int(audio_merge_size)
+ )
+ total = 0
+ for start in range(0, num_samples, n_samples):
+ chunk_samples = min(n_samples, num_samples - start)
+ total += (chunk_samples - 1) // stride + 1
+ return total
+
+
+def _get_max_audio_samples(feature_extractor: Any) -> int:
+ if hasattr(feature_extractor, "chunk_length"):
+ return int(feature_extractor.chunk_length * feature_extractor.sampling_rate)
+ return int(feature_extractor.n_samples)
+
+
+def _as_audio_embedding_list(audio_embeds: object) -> list[torch.Tensor]:
+ if isinstance(audio_embeds, torch.Tensor):
+ if audio_embeds.ndim == 2:
+ return [audio_embeds]
+ if audio_embeds.ndim == 3:
+ return list(audio_embeds.unbind(dim=0))
+ raise ValueError(
+ f"`audio_embeds` must be a 2D or 3D tensor, got {audio_embeds.ndim}D."
+ )
+
+ if isinstance(audio_embeds, (list, tuple)) and all(
+ isinstance(audio_embed, torch.Tensor) for audio_embed in audio_embeds
+ ):
+ return list(audio_embeds)
+
+ raise TypeError(
+ "`audio_embeds` must be a torch.Tensor or a list of torch.Tensor "
+ f"objects, got {type(audio_embeds)!r}."
+ )
+
+
+def _get_required_token_id(tokenizer: Any, token: str) -> int:
+ token_id = tokenizer.convert_tokens_to_ids(token)
+ if token_id is None or (isinstance(token_id, int) and token_id < 0):
+ raise ValueError(f"Tokenizer is missing required token {token!r}.")
+ return int(token_id)
+
+
+def _get_audios_from_mm_data(mm_data: Mapping[str, object]) -> list[Any]:
+ mm_data_dict = dict(mm_data)
+ audio_data = mm_data_dict.pop("audios", mm_data_dict.pop("audio", []))
+ if isinstance(audio_data, list):
+ audios = audio_data
+ elif isinstance(audio_data, tuple) and not (
+ len(audio_data) == 2 and isinstance(audio_data[1], (int, float))
+ ):
+ audios = list(audio_data)
+ elif audio_data is None:
+ audios = []
+ else:
+ audios = [audio_data]
+
+ audio_arrays: list[Any] = []
+ for audio in audios:
+ if isinstance(audio, (tuple, list)) and len(audio) == 2:
+ audio = audio[0]
+ audio_arrays.append(audio)
+ return audio_arrays
+
+
+def _add_vllm_audio_metadata(
+ processed: BatchFeature,
+ num_audios: int,
+) -> BatchFeature:
+ audio_feature_lengths = processed["audio_feature_lengths"]
+ if not isinstance(audio_feature_lengths, torch.Tensor):
+ audio_feature_lengths = torch.tensor(audio_feature_lengths, dtype=torch.long)
+ audio_feature_lengths = audio_feature_lengths.to(dtype=torch.long)
+
+ audio_chunk_mapping = processed.get("audio_chunk_mapping")
+ if audio_chunk_mapping is None:
+ if num_audios == 1:
+ audio_chunk_mapping = torch.zeros_like(audio_feature_lengths)
+ elif audio_feature_lengths.numel() == num_audios:
+ audio_chunk_mapping = torch.arange(num_audios, dtype=torch.long)
+ else:
+ raise ValueError(
+ "The MOSS processor did not return `audio_chunk_mapping`, and "
+ "the chunk-to-audio mapping cannot be inferred."
+ )
+ elif not isinstance(audio_chunk_mapping, torch.Tensor):
+ audio_chunk_mapping = torch.tensor(audio_chunk_mapping, dtype=torch.long)
+ audio_chunk_mapping = audio_chunk_mapping.to(dtype=torch.long)
+ if audio_chunk_mapping.numel() != audio_feature_lengths.numel():
+ raise ValueError(
+ "`audio_chunk_mapping` must contain one item per audio chunk: got "
+ f"{audio_chunk_mapping.numel()} mappings for "
+ f"{audio_feature_lengths.numel()} chunks."
+ )
+ if audio_chunk_mapping.numel() > 0 and (
+ audio_chunk_mapping.min().item() < 0
+ or audio_chunk_mapping.max().item() >= num_audios
+ ):
+ raise ValueError("`audio_chunk_mapping` contains an out-of-range audio index.")
+
+ audio_chunk_counts = torch.bincount(
+ audio_chunk_mapping.cpu(),
+ minlength=num_audios,
+ ).to(dtype=torch.long)
+ audio_token_lengths = torch.zeros(
+ num_audios,
+ dtype=torch.long,
+ device=audio_feature_lengths.device,
+ )
+ audio_token_lengths.scatter_add_(
+ 0,
+ audio_chunk_mapping.to(device=audio_feature_lengths.device),
+ audio_feature_lengths,
+ )
+
+ processed["audio_feature_lengths"] = audio_feature_lengths
+ processed["audio_chunk_counts"] = audio_chunk_counts
+ processed["audio_token_lengths"] = audio_token_lengths
+ return processed
+
+
+class MossTranscribeDiarizeWhisperEncoder(WhisperEncoder):
+ packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_substr={".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."},
+ orig_to_new_stacked={
+ ".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
+ ".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
+ ".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
+ },
+ )
+
+ def __init__(
+ self,
+ *,
+ vllm_config: VllmConfig,
+ prefix: str = "",
+ ) -> None:
+ super().__init__(
+ vllm_config=vllm_config.with_hf_config(
+ vllm_config.model_config.hf_config.audio_config
+ ),
+ prefix=prefix,
+ )
+ self.audio_merge_size = int(vllm_config.model_config.hf_config.audio_merge_size)
+ self.max_encoder_batch: int | None = None
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight")
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+
+ def forward(
+ self,
+ input_features: torch.Tensor,
+ audio_feature_lengths: torch.Tensor,
+ ) -> torch.Tensor:
+ if input_features.numel() == 0:
+ return input_features.new_empty((1, 0, self.conv1.out_channels))
+ device = self.conv1.weight.device
+ dtype = self.conv1.weight.dtype
+ input_features = input_features.to(device=device, dtype=dtype)
+ audio_feature_lengths = audio_feature_lengths.to(device=device)
+ batch_size = self.max_encoder_batch or input_features.shape[0]
+ encoded_parts: list[torch.Tensor] = []
+
+ for start in range(0, input_features.shape[0], batch_size):
+ encoded_parts.append(
+ super().forward([input_features[start : start + batch_size]])
+ )
+
+ hidden = torch.cat(encoded_parts, dim=0)
+ chunks = [
+ hidden[idx : idx + 1, : int(token_len.item()) * self.audio_merge_size]
+ for idx, token_len in enumerate(audio_feature_lengths)
+ ]
+ return torch.cat(chunks, dim=1)
+
+
+class MossTranscribeDiarizeVQAdaptor(nn.Module):
+ def __init__(self, input_dim: int, hidden_size: int, eps: float) -> None:
+ super().__init__()
+ self.layers = nn.Sequential(
+ nn.Linear(input_dim, hidden_size, bias=True),
+ nn.SiLU(),
+ nn.Linear(hidden_size, hidden_size, bias=True),
+ nn.LayerNorm(hidden_size, eps=eps, bias=True),
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.layers(x)
+
+
+def _mtd_field_config(
+ hf_inputs: Mapping[str, torch.Tensor],
+) -> Mapping[str, MultiModalFieldConfig]:
+ fields: dict[str, MultiModalFieldConfig] = {}
+ if "audio_embeds" in hf_inputs:
+ fields["audio_embeds"] = MultiModalFieldConfig.batched("audio")
+ if "audio_chunk_counts" in hf_inputs:
+ audio_chunk_counts = hf_inputs["audio_chunk_counts"]
+ fields.update(
+ input_features=MultiModalFieldConfig.flat_from_sizes(
+ "audio",
+ audio_chunk_counts,
+ ),
+ audio_feature_lengths=MultiModalFieldConfig.flat_from_sizes(
+ "audio",
+ audio_chunk_counts,
+ ),
+ audio_chunk_counts=MultiModalFieldConfig.batched("audio"),
+ audio_token_lengths=MultiModalFieldConfig.batched("audio"),
+ )
+ return fields
+
+
+class MossTranscribeDiarizeMultiModalDataParser(MultiModalDataParser):
+ def _parse_audio_data(
+ self,
+ data: dict[str, torch.Tensor] | ModalityData[AudioItem],
+ ) -> ModalityDataItems[Any, Any] | None:
+ if isinstance(data, dict):
+ return DictEmbeddingItems(
+ data,
+ modality="audio",
+ required_fields={"audio_embeds"},
+ fields_factory=_mtd_field_config,
+ )
+
+ return super()._parse_audio_data(data)
+
+
+class MossTranscribeDiarizeProcessingInfo(BaseProcessingInfo):
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+ return {"audio": 1}
+
+ def get_hf_processor(self, **kwargs: object) -> Any:
+ return self.ctx.get_hf_processor(**kwargs)
+
+ def get_feature_extractor(self, **kwargs: object) -> Any:
+ return self.get_hf_processor(**kwargs).feature_extractor
+
+ def get_data_parser(self) -> MultiModalDataParser:
+ feature_extractor = self.get_feature_extractor()
+ return MossTranscribeDiarizeMultiModalDataParser(
+ target_sr=feature_extractor.sampling_rate,
+ target_channels=1,
+ expected_hidden_size=self._get_expected_hidden_size(),
+ )
+
+ def get_mm_max_tokens_per_item(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ ) -> Mapping[str, int]:
+ if mm_counts.get("audio", 0) <= 0:
+ return {}
+
+ feature_extractor = self.get_feature_extractor()
+ max_audio_samples = _get_max_audio_samples(feature_extractor)
+ max_audio_tokens = _compute_total_audio_tokens(
+ max_audio_samples,
+ feature_extractor,
+ self.get_hf_processor().audio_merge_size,
+ )
+ return {"audio": min(seq_len, max_audio_tokens)}
+
+
+class MossTranscribeDiarizeDummyInputsBuilder(
+ BaseDummyInputsBuilder[MossTranscribeDiarizeProcessingInfo]
+):
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+ return AUDIO_PLACEHOLDER * mm_counts.get("audio", 0)
+
+ def get_dummy_mm_data(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ mm_options: Mapping[str, BaseDummyOptions],
+ ) -> MultiModalDataDict:
+ num_audios = mm_counts.get("audio", 0)
+ if num_audios == 0:
+ return {}
+
+ feature_extractor = self.info.get_feature_extractor()
+ return {
+ "audio": self._get_dummy_audios(
+ length=_get_max_audio_samples(feature_extractor),
+ num_audios=num_audios,
+ overrides=mm_options.get("audio"),
+ )
+ }
+
+ def get_dummy_processor_inputs(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ mm_options: Mapping[str, BaseDummyOptions],
+ ) -> ProcessorInputs:
+ dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
+ num_audios = mm_counts.get("audio", 0)
+ tokenizer = self.info.get_tokenizer()
+ prompt = tokenizer.encode(
+ AUDIO_PLACEHOLDER * num_audios,
+ add_special_tokens=False,
+ ) or tokenizer.encode(
+ "\n",
+ add_special_tokens=False,
+ )
+ return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items)
+
+
+class MossTranscribeDiarizeMultiModalProcessor(
+ BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
+):
+ def _call_hf_processor(
+ self,
+ prompt: str,
+ mm_data: Mapping[str, object],
+ mm_kwargs: Mapping[str, object],
+ tok_kwargs: Mapping[str, object],
+ ) -> BatchFeature:
+ tokenizer = self.info.get_tokenizer()
+ audios = _get_audios_from_mm_data(mm_data)
+ if not audios:
+ input_ids = tokenizer.encode(
+ prompt,
+ add_special_tokens=tok_kwargs.get("add_special_tokens", False),
+ )
+ return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
+
+ processed = self.info.ctx.call_hf_processor(
+ self.info.get_hf_processor(**mm_kwargs),
+ dict(text=prompt, audio=audios),
+ dict(**mm_kwargs, **tok_kwargs),
+ )
+ return _add_vllm_audio_metadata(processed, len(audios))
+
+ def _hf_processor_applies_updates(
+ self,
+ prompt_text: str,
+ mm_items: MultiModalDataItems,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ tokenization_kwargs: Mapping[str, object],
+ ) -> bool:
+ return mm_items.get_count("audio", strict=False) > 0
+
+ def _get_mm_fields_config(
+ self,
+ hf_inputs: BatchFeature,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ ) -> Mapping[str, MultiModalFieldConfig]:
+ return _mtd_field_config(hf_inputs)
+
+ def _get_prompt_updates(
+ self,
+ mm_items: MultiModalDataItems,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ out_mm_kwargs: MultiModalKwargsItems,
+ ) -> Sequence[PromptUpdate]:
+ out_mm_data = out_mm_kwargs.get_data()
+ audio_token_lengths_tensor = out_mm_data.get("audio_token_lengths")
+ if audio_token_lengths_tensor is None:
+ audio_embeds = out_mm_data.get("audio_embeds")
+ if audio_embeds is None:
+ audio_token_lengths: list[int] = []
+ else:
+ audio_token_lengths = [
+ int(audio_embed.shape[0])
+ for audio_embed in _as_audio_embedding_list(audio_embeds)
+ ]
+ else:
+ if not isinstance(audio_token_lengths_tensor, torch.Tensor):
+ raise TypeError(
+ "`audio_token_lengths` must be a torch.Tensor, got "
+ f"{type(audio_token_lengths_tensor)!r}."
+ )
+ audio_token_lengths = [
+ int(length) for length in audio_token_lengths_tensor.tolist()
+ ]
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+ tokenizer = self.info.get_tokenizer()
+ audio_start_id = _get_required_token_id(tokenizer, processor.audio_start_token)
+ audio_token_id = int(processor.audio_token_id)
+ audio_end_id = _get_required_token_id(tokenizer, processor.audio_end_token)
+
+ def get_num_tokens(item_idx: int) -> int:
+ if item_idx >= len(audio_token_lengths):
+ raise ValueError(
+ "Cannot determine the number of audio tokens for audio item "
+ f"{item_idx}."
+ )
+ num_tokens = audio_token_lengths[item_idx]
+ if num_tokens <= 0:
+ raise ValueError("Audio input is too short to produce any tokens.")
+ return num_tokens
+
+ def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
+ num_tokens = get_num_tokens(item_idx)
+ audio_tokens = processor._audio_span_ids(num_tokens)
+ return PromptUpdateDetails.select_token_id(
+ [audio_start_id] + audio_tokens + [audio_end_id],
+ embed_token_id=audio_token_id,
+ )
+
+ return [
+ PromptReplacement(
+ modality="audio",
+ target=AUDIO_PLACEHOLDER,
+ replacement=get_replacement,
+ ),
+ ]
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+ MossTranscribeDiarizeMultiModalProcessor,
+ info=MossTranscribeDiarizeProcessingInfo,
+ dummy_inputs=MossTranscribeDiarizeDummyInputsBuilder,
+)
+class MossTranscribeDiarizeForConditionalGeneration(
+ nn.Module,
+ SupportsMultiModal,
+ SupportsPP,
+ SupportsTranscription,
+):
+ supports_transcription = True
+ supports_transcription_only = True
+ supports_segment_timestamp = False
+ supported_languages = ISO639_1_SUPPORTED_LANGS
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_prefix={
+ "language_model.layers.": "language_model.model.layers.",
+ "language_model.embed_tokens.": "language_model.model.embed_tokens.",
+ "language_model.norm.": "language_model.model.norm.",
+ "model.language_model.model.": "language_model.model.",
+ "model.language_model.lm_head.": "language_model.lm_head.",
+ "model.language_model.": "language_model.model.",
+ "model.whisper_encoder.": "whisper_encoder.",
+ "model.vq_adaptor.": "vq_adaptor.",
+ "model.": None,
+ },
+ )
+
+ @classmethod
+ def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+ return AUDIO_PLACEHOLDER if modality.startswith("audio") else None
+
+ @classmethod
+ def get_speech_to_text_config(
+ cls,
+ model_config: ModelConfig,
+ task_type: str,
+ ) -> SpeechToTextConfig:
+ processor = cached_processor_from_config(model_config)
+ return SpeechToTextConfig(
+ max_audio_clip_s=None,
+ sample_rate=processor.feature_extractor.sampling_rate,
+ min_energy_split_window_size=None,
+ )
+
+ @classmethod
+ def get_num_audio_tokens(
+ cls,
+ audio_duration_s: float,
+ stt_config: SpeechToTextConfig,
+ model_config: ModelConfig,
+ ) -> int | None:
+ processor = cached_processor_from_config(model_config)
+ num_samples = math.ceil(audio_duration_s * stt_config.sample_rate)
+ return _compute_total_audio_tokens(
+ num_samples,
+ processor.feature_extractor,
+ processor.audio_merge_size,
+ )
+
+ @classmethod
+ def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
+ stt_config = stt_params.stt_config
+ question = stt_params.request_prompt or DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT
+ question = question.strip() or DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT
+ prompt = (
+ "<|im_start|>system\n"
+ "You are a helpful assistant.<|im_end|>\n"
+ f"<|im_start|>user\n{AUDIO_PLACEHOLDER}\n"
+ f"{question}<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ )
+ return TextPrompt(
+ prompt=prompt,
+ multi_modal_data={"audio": (stt_params.audio, stt_config.sample_rate)},
+ )
+
+ @classmethod
+ def post_process_output(cls, text: str) -> str:
+ return text.strip()
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+ super().__init__()
+ self.config = vllm_config.model_config.hf_config
+ self.dtype = vllm_config.model_config.dtype
+
+ with self._mark_tower_model(vllm_config, "audio"):
+ self.whisper_encoder = MossTranscribeDiarizeWhisperEncoder(
+ vllm_config=vllm_config,
+ prefix=maybe_prefix(prefix, "whisper_encoder"),
+ )
+ self.vq_adaptor = MossTranscribeDiarizeVQAdaptor(
+ input_dim=int(self.config.adaptor_input_dim),
+ hidden_size=int(self.config.text_config.hidden_size),
+ eps=float(self.config.text_config.rms_norm_eps),
+ )
+
+ with self._mark_language_model(vllm_config):
+ self.language_model = init_vllm_registered_model(
+ vllm_config=vllm_config,
+ hf_config=self.config.text_config,
+ prefix=maybe_prefix(prefix, "language_model"),
+ architectures=["Qwen3ForCausalLM"],
+ )
+
+ self.make_empty_intermediate_tensors = (
+ self.language_model.make_empty_intermediate_tensors
+ )
+
+ def _time_merge(self, features: torch.Tensor) -> torch.Tensor:
+ batch, seq_len, dim = features.shape
+ merge_size = int(self.config.audio_merge_size)
+ seq_len_trim = (seq_len // merge_size) * merge_size
+ return features[:, :seq_len_trim, :].reshape(
+ batch,
+ seq_len_trim // merge_size,
+ dim * merge_size,
+ )
+
+ def _parse_and_validate_audio_input(
+ self,
+ **kwargs: object,
+ ) -> MossTranscribeDiarizeInputs | None:
+ input_features = kwargs.pop("input_features", None)
+ audio_embeds = kwargs.pop("audio_embeds", None)
+ audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
+ audio_chunk_counts = kwargs.pop("audio_chunk_counts", None)
+ if input_features is None and audio_embeds is None:
+ return None
+ if audio_embeds is not None:
+ return MossTranscribeDiarizeEmbeddingInputs(
+ type="audio_embeds",
+ audio_embeds=_as_audio_embedding_list(audio_embeds),
+ )
+ return MossTranscribeDiarizeAudioInputs(
+ type="audio_features",
+ input_features=input_features,
+ audio_feature_lengths=audio_feature_lengths,
+ audio_chunk_counts=audio_chunk_counts,
+ )
+
+ def _process_audio_input(
+ self, audio_input: MossTranscribeDiarizeInputs
+ ) -> list[torch.Tensor]:
+ if audio_input["type"] == "audio_embeds":
+ return list(audio_input["audio_embeds"])
+
+ input_features = audio_input["input_features"]
+ audio_feature_lengths = audio_input["audio_feature_lengths"]
+ audio_chunk_counts = audio_input["audio_chunk_counts"]
+ if input_features is None or audio_feature_lengths is None:
+ raise ValueError(
+ "MOSS-Transcribe-Diarize audio inputs require both "
+ "`input_features` and `audio_feature_lengths`."
+ )
+ if audio_feature_lengths.numel() != input_features.shape[0]:
+ raise ValueError(
+ "`audio_feature_lengths` must contain one length per "
+ "`input_features` chunk: got "
+ f"{audio_feature_lengths.numel()} lengths for "
+ f"{input_features.shape[0]} chunks."
+ )
+ if audio_chunk_counts is None:
+ audio_chunk_counts = audio_feature_lengths.new_tensor(
+ [input_features.shape[0]],
+ dtype=torch.long,
+ )
+ else:
+ audio_chunk_counts = audio_chunk_counts.to(dtype=torch.long)
+ if audio_chunk_counts.numel() == 0:
+ raise ValueError("`audio_chunk_counts` must contain at least one item.")
+ if torch.any(audio_chunk_counts <= 0):
+ raise ValueError("`audio_chunk_counts` must contain positive counts.")
+ num_audio_chunks = int(audio_chunk_counts.sum().item())
+ if num_audio_chunks != input_features.shape[0]:
+ raise ValueError(
+ "`audio_chunk_counts` must sum to the number of input feature chunks: "
+ f"got {num_audio_chunks} chunks for "
+ f"{input_features.shape[0]} input chunks."
+ )
+
+ features = self.whisper_encoder(input_features, audio_feature_lengths)
+ merged = self._time_merge(features.to(dtype=self.dtype))
+ projected = self.vq_adaptor(merged).squeeze(0)
+
+ audio_chunk_offsets = torch.cumsum(audio_chunk_counts, dim=0)
+ audio_chunk_offsets = torch.cat(
+ [audio_chunk_offsets.new_zeros(1), audio_chunk_offsets]
+ )
+ tokens_per_item = [
+ int(audio_feature_lengths[start:end].sum().item())
+ for start, end in zip(
+ audio_chunk_offsets[:-1].tolist(),
+ audio_chunk_offsets[1:].tolist(),
+ )
+ ]
+ if any(num_tokens <= 0 for num_tokens in tokens_per_item):
+ raise ValueError("Audio input is too short to produce any tokens.")
+ return list(projected.split(tokens_per_item, dim=0))
+
+ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
+ if audio_input is None:
+ return []
+ return self._process_audio_input(audio_input)
+
+ def embed_input_ids(
+ self,
+ input_ids: torch.Tensor,
+ multimodal_embeddings: MultiModalEmbeddings | None = None,
+ *,
+ is_multimodal: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ inputs_embeds = self.language_model.embed_input_ids(input_ids)
+ if not multimodal_embeddings:
+ return inputs_embeds
+ return _merge_multimodal_embeddings(
+ inputs_embeds=inputs_embeds,
+ multimodal_embeddings=multimodal_embeddings,
+ is_multimodal=_require_is_multimodal(is_multimodal),
+ )
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ **kwargs: object,
+ ) -> torch.Tensor | IntermediateTensors:
+ if intermediate_tensors is not None:
+ inputs_embeds = None
+ return self.language_model(
+ input_ids=input_ids,
+ positions=positions,
+ intermediate_tensors=intermediate_tensors,
+ inputs_embeds=inputs_embeds,
+ )
+
+ def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
+ return self.language_model.compute_logits(hidden_states)
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(
+ weights,
+ mapper=self.hf_to_vllm_mapper,
+ )
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 07edf2a62d..c9e43ed635 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -447,6 +447,10 @@ _MULTIMODAL_MODELS = {
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
+ "MossTranscribeDiarizeForConditionalGeneration": (
+ "moss_transcribe_diarize",
+ "MossTranscribeDiarizeForConditionalGeneration",
+ ),
"LightOnOCRForConditionalGeneration": (
"lightonocr",
"LightOnOCRForConditionalGeneration",
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index 37b740abdb..a848bbe142 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -107,6 +107,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
minimax_m3_vl="MiniMaxM3Config",
minimax_m3_mtp="MiniMaxM3MTPConfig",
moondream3="Moondream3Config",
+ moss_transcribe_diarize="MossTranscribeDiarizeConfig",
eagle="EAGLEConfig",
speculators="SpeculatorsConfig",
nemotron="NemotronConfig",
diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py
index e1050d6a04..f48b0dd6df 100644
--- a/vllm/transformers_utils/configs/__init__.py
+++ b/vllm/transformers_utils/configs/__init__.py
@@ -61,6 +61,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
"Moondream3Config": "vllm.transformers_utils.configs.moondream3",
"Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3",
"Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3",
+ "MossTranscribeDiarizeConfig": (
+ "vllm.transformers_utils.configs.moss_transcribe_diarize"
+ ),
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
@@ -136,6 +139,7 @@ __all__ = [
"Moondream3Config",
"Moondream3TextConfig",
"Moondream3VisionConfig",
+ "MossTranscribeDiarizeConfig",
"MoonViTConfig",
"KimiLinearConfig",
"KimiVLConfig",
diff --git a/vllm/transformers_utils/configs/moss_transcribe_diarize.py b/vllm/transformers_utils/configs/moss_transcribe_diarize.py
new file mode 100644
index 0000000000..20abbd8fee
--- /dev/null
+++ b/vllm/transformers_utils/configs/moss_transcribe_diarize.py
@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+from transformers import PretrainedConfig, Qwen3Config
+from transformers.models.whisper.configuration_whisper import WhisperConfig
+
+
+class MossTranscribeDiarizeConfig(PretrainedConfig):
+ """Configuration for MOSS-Transcribe-Diarize."""
+
+ model_type = "moss_transcribe_diarize"
+ sub_configs = {"text_config": Qwen3Config, "audio_config": WhisperConfig}
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ text_config: dict[str, Any] | Qwen3Config | None = None,
+ audio_config: dict[str, Any] | WhisperConfig | None = None,
+ audio_token_id: int = 151671,
+ audio_merge_size: int = 4,
+ adaptor_input_dim: int | None = None,
+ tie_word_embeddings: bool = True,
+ **kwargs: Any,
+ ) -> None:
+ text_config_obj: Qwen3Config
+ if text_config is None:
+ text_config_obj = Qwen3Config(
+ vocab_size=151936,
+ hidden_size=1024,
+ intermediate_size=3072,
+ num_hidden_layers=28,
+ num_attention_heads=16,
+ num_key_value_heads=8,
+ head_dim=128,
+ max_position_embeddings=40960,
+ tie_word_embeddings=tie_word_embeddings,
+ rope_theta=1_000_000.0,
+ layer_types=["full_attention"] * 28,
+ )
+ elif isinstance(text_config, dict):
+ text_config_obj = Qwen3Config(**text_config)
+ else:
+ text_config_obj = text_config
+
+ audio_config_obj: WhisperConfig
+ if audio_config is None:
+ audio_config_obj = WhisperConfig(
+ num_mel_bins=80,
+ d_model=1024,
+ encoder_layers=24,
+ encoder_attention_heads=16,
+ encoder_ffn_dim=4096,
+ max_source_positions=1500,
+ dropout=0.0,
+ attention_dropout=0.0,
+ activation_dropout=0.0,
+ activation_function="gelu",
+ encoder_layerdrop=0.0,
+ scale_embedding=False,
+ )
+ elif isinstance(audio_config, dict):
+ audio_config_obj = WhisperConfig(**audio_config)
+ else:
+ audio_config_obj = audio_config
+
+ text_config_obj.tie_word_embeddings = tie_word_embeddings
+ if not getattr(text_config_obj, "layer_types", None):
+ text_config_obj.layer_types = [
+ "full_attention"
+ ] * text_config_obj.num_hidden_layers
+
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+ self.text_config = text_config_obj
+ self.audio_config = audio_config_obj
+ self.audio_token_id = int(audio_token_id)
+ self.audio_merge_size = int(audio_merge_size)
+ self.adaptor_input_dim = (
+ int(adaptor_input_dim)
+ if adaptor_input_dim is not None
+ else int(audio_config_obj.d_model) * int(audio_merge_size)
+ )
+
+ self.vocab_size = int(text_config_obj.vocab_size)
+ self.hidden_size = int(text_config_obj.hidden_size)
+ self.intermediate_size = int(text_config_obj.intermediate_size)
+ self.num_hidden_layers = int(text_config_obj.num_hidden_layers)
+ self.num_attention_heads = int(text_config_obj.num_attention_heads)
+ self.num_key_value_heads = int(text_config_obj.num_key_value_heads)
+ self.head_dim = int(text_config_obj.head_dim)
+ self.hidden_act = text_config_obj.hidden_act
+ self.max_position_embeddings = int(text_config_obj.max_position_embeddings)
+ self.rms_norm_eps = float(text_config_obj.rms_norm_eps)
+ rope_parameters = getattr(text_config_obj, "rope_parameters", None)
+ rope_theta = float(getattr(text_config_obj, "rope_theta", 1_000_000.0))
+ if rope_parameters is None:
+ rope_parameters = {
+ "rope_type": "default",
+ "rope_theta": rope_theta,
+ }
+ text_config_obj.rope_parameters = rope_parameters
+ self.rope_parameters = rope_parameters
+ self.rope_theta = float(rope_parameters.get("rope_theta", rope_theta))
+ self.attention_bias = bool(text_config_obj.attention_bias)
+ self.attention_dropout = float(text_config_obj.attention_dropout)
+ self.is_causal = True