diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 74e9e7739f..59854fe6dc 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -598,6 +598,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I+ | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
| `MolmoForCausalLM` | Molmo | T + I+ | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ |
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`^, `allenai/MolmoWeb-8B`^ | ✅︎ | ✅︎ |
+| `MossAudioModel` | MOSS-Audio | T + A+ | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
| `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
diff --git a/tests/benchmarks/test_throughput_cli.py b/tests/benchmarks/test_throughput_cli.py
index a579b59e8a..87a8cecd5e 100644
--- a/tests/benchmarks/test_throughput_cli.py
+++ b/tests/benchmarks/test_throughput_cli.py
@@ -4,6 +4,13 @@ import subprocess
import pytest
+from vllm.benchmarks.datasets import SampleRequest
+from vllm.benchmarks.throughput import (
+ _run_vllm_chat_requests,
+ add_cli_args,
+)
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
@@ -28,3 +35,70 @@ def test_bench_throughput():
print(result.stderr)
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
+
+
+def test_bench_throughput_accepts_custom_audio_args():
+ parser = FlexibleArgumentParser()
+ add_cli_args(parser)
+
+ args = parser.parse_args(
+ [
+ "--dataset-name",
+ "custom_audio",
+ "--dataset-path",
+ "audio.jsonl",
+ "--no-oversample",
+ "--custom-output-len",
+ "32",
+ "--enable-multimodal-chat",
+ ]
+ )
+
+ assert args.dataset_name == "custom_audio"
+ assert args.no_oversample
+ assert args.custom_output_len == 32
+ assert args.enable_multimodal_chat
+
+
+def test_vllm_chat_requests_include_multimodal_content():
+ class FakeLLM:
+ def __init__(self):
+ self.prompts = None
+
+ def chat(self, prompts, sampling_params, use_tqdm):
+ del sampling_params, use_tqdm
+ self.prompts = prompts
+ return []
+
+ llm = FakeLLM()
+ audio_content = {
+ "type": "input_audio",
+ "input_audio": {"data": "abc", "format": "wav"},
+ }
+ request = SampleRequest(
+ prompt="Transcribe this audio.",
+ prompt_len=1,
+ expected_output_len=8,
+ multi_modal_data=audio_content,
+ )
+
+ _run_vllm_chat_requests(
+ llm,
+ [request],
+ n=1,
+ disable_detokenize=False,
+ do_profile=False,
+ prequeue_requests=False,
+ )
+
+ assert llm.prompts == [
+ [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Transcribe this audio."},
+ audio_content,
+ ],
+ }
+ ]
+ ]
diff --git a/tests/entrypoints/openai/chat_completion/test_chat.py b/tests/entrypoints/openai/chat_completion/test_chat.py
index 16a3cd857c..dbfb48f235 100644
--- a/tests/entrypoints/openai/chat_completion/test_chat.py
+++ b/tests/entrypoints/openai/chat_completion/test_chat.py
@@ -824,7 +824,10 @@ async def test_invocations(server: RemoteOpenAIServer, client: openai.AsyncOpenA
chat_output = chat_response.json()
invocation_output = invocation_response.json()
- assert chat_output.keys() == invocation_output.keys()
+ extra_keys = invocation_output.keys() - chat_output.keys()
+ missing_keys = chat_output.keys() - invocation_output.keys()
+ assert missing_keys == set()
+ assert extra_keys <= {"moderation"}
assert chat_output["choices"] == invocation_output["choices"]
diff --git a/tests/models/multimodal/generation/test_moss_audio.py b/tests/models/multimodal/generation/test_moss_audio.py
new file mode 100644
index 0000000000..a9da471b2c
--- /dev/null
+++ b/tests/models/multimodal/generation/test_moss_audio.py
@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from vllm.assets.audio import AudioAsset
+from vllm.model_executor.models.moss_audio import MOSS_AUDIO_PLACEHOLDER
+from vllm.platforms import current_platform
+
+from ...registry import HF_EXAMPLE_MODELS
+from ...utils import check_logprobs_close
+
+CORE_MODEL = pytest.param(
+ "OpenMOSS-Team/MOSS-Audio-4B-Instruct",
+ marks=pytest.mark.core_model,
+ id="4b-instruct",
+)
+
+EXTENDED_MODELS = [
+ "OpenMOSS-Team/MOSS-Audio-4B-Thinking",
+ "OpenMOSS-Team/MOSS-Audio-8B-Instruct",
+ "OpenMOSS-Team/MOSS-Audio-8B-Thinking",
+]
+
+ACCURACY_MODELS = [CORE_MODEL, *EXTENDED_MODELS]
+
+PARALLEL_SMOKE_CASES = [
+ pytest.param({"tensor_parallel_size": 2}, id="tp2"),
+ pytest.param({"pipeline_parallel_size": 2}, id="pp2"),
+ pytest.param(
+ {"tensor_parallel_size": 2, "pipeline_parallel_size": 2},
+ id="tp2_pp2",
+ ),
+]
+
+HF_ACCURACY_SKIP_REASON = (
+ "HF AutoModelForCausalLM cannot load remote MOSS-Audio configs; "
+ "vLLM generation coverage is provided by the smoke tests below."
+)
+
+
+@pytest.mark.core_model
+def test_moss_audio_generation_smoke(vllm_runner) -> None:
+ model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+
+ prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
+ audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
+
+ with vllm_runner(
+ model,
+ dtype="half",
+ enforce_eager=True,
+ max_model_len=1024,
+ limit_mm_per_prompt={"audio": 1},
+ trust_remote_code=True,
+ ) as vllm_model:
+ outputs = vllm_model.generate_greedy(
+ prompts,
+ max_tokens=4,
+ audios=audios,
+ )
+
+ assert len(outputs) == 1
+ assert len(outputs[0][1]) > 0
+
+
+@pytest.mark.skip(reason=HF_ACCURACY_SKIP_REASON)
+@pytest.mark.parametrize("model", ACCURACY_MODELS)
+@pytest.mark.parametrize("dtype", ["half"])
+@pytest.mark.parametrize("max_tokens", [8])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_moss_audio_hf_vllm_accuracy(
+ hf_runner,
+ vllm_runner,
+ model: str,
+ dtype: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+
+ prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."]
+ audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
+
+ with vllm_runner(
+ model,
+ dtype=dtype,
+ enforce_eager=True,
+ max_model_len=1024,
+ limit_mm_per_prompt={"audio": 1},
+ trust_remote_code=True,
+ ) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ prompts,
+ max_tokens,
+ num_logprobs=num_logprobs,
+ audios=audios,
+ )
+
+ with hf_runner(model, dtype=dtype, trust_remote_code=True) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ prompts,
+ max_tokens,
+ num_logprobs=num_logprobs,
+ audios=audios,
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+
+
+@pytest.mark.core_model
+@pytest.mark.parametrize("parallel_kwargs", PARALLEL_SMOKE_CASES)
+def test_moss_audio_parallel_smoke(vllm_runner, parallel_kwargs) -> None:
+ model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
+ required_gpus = parallel_kwargs.get(
+ "tensor_parallel_size", 1
+ ) * parallel_kwargs.get("pipeline_parallel_size", 1)
+ if current_platform.device_count() < required_gpus:
+ # TP/PP integration smoke runs on local or multi-GPU CI only.
+ pytest.skip(f"Requires at least {required_gpus} GPUs")
+
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+
+ prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
+ audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
+
+ with vllm_runner(
+ model,
+ dtype="half",
+ enforce_eager=True,
+ max_model_len=1024,
+ limit_mm_per_prompt={"audio": 1},
+ trust_remote_code=True,
+ **parallel_kwargs,
+ ) as vllm_model:
+ outputs = vllm_model.generate_greedy(
+ prompts,
+ max_tokens=4,
+ audios=audios,
+ )
+
+ assert len(outputs) == 1
+ assert len(outputs[0][1]) > 0
diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py
index 26ab67e5d9..ea5aeb8e2c 100644
--- a/tests/models/multimodal/processing/test_common.py
+++ b/tests/models/multimodal/processing/test_common.py
@@ -432,6 +432,12 @@ def test_processing_correctness(
)
if model_id == "CohereLabs/cohere-transcribe-03-2026":
pytest.skip("Fix later")
+ if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
+ pytest.skip(
+ "MOSS-Audio uses a custom processor that dynamically expands "
+ "audio placeholders from processed audio lengths. Its vLLM "
+ "processor paths are covered by test_moss_audio.py."
+ )
_test_processing_correctness(
model_id,
diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py
new file mode 100644
index 0000000000..d5c573c4ff
--- /dev/null
+++ b/tests/models/multimodal/processing/test_moss_audio.py
@@ -0,0 +1,694 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+from transformers import Qwen3Config
+
+from vllm.model_executor.models.interfaces import SupportsLoRA, supports_lora
+from vllm.model_executor.models.moss_audio import (
+ MOSS_AUDIO_BOS_TOKEN,
+ MOSS_AUDIO_BOS_TOKEN_ID,
+ MOSS_AUDIO_EOS_TOKEN,
+ MOSS_AUDIO_EOS_TOKEN_ID,
+ MOSS_AUDIO_PLACEHOLDER,
+ MOSS_AUDIO_TOKEN,
+ MOSS_AUDIO_TOKEN_ID,
+ GatedMLP,
+ MossAudioConfig,
+ MossAudioDummyInputsBuilder,
+ MossAudioEncoder,
+ MossAudioEncoderConfig,
+ MossAudioModel,
+ MossAudioMultiModalProcessor,
+ MossAudioProcessingInfo,
+ MossAudioProcessor,
+ MossQwen3ForCausalLM,
+ MossQwen3Model,
+)
+from vllm.model_executor.models.utils import AutoWeightsLoader
+from vllm.multimodal.cache import MultiModalProcessorOnlyCache
+from vllm.multimodal.inputs import batched_tensors_equal
+from vllm.sequence import IntermediateTensors
+
+
+class _Tokenizer:
+ def encode(self, text, add_special_tokens=False):
+ del add_special_tokens
+ return [ord(char) for char in text]
+
+ def decode(self, token_ids, **kwargs):
+ del kwargs
+ return "".join(chr(token_id) for token_id in token_ids)
+
+ def batch_decode(self, batch_token_ids, **kwargs):
+ return [self.decode(token_ids, **kwargs) for token_ids in batch_token_ids]
+
+
+class _MMConfig:
+ enable_mm_embeds = False
+ mm_processor_cache_gb = 1
+
+ def merge_mm_processor_kwargs(self, kwargs):
+ return dict(kwargs)
+
+ def get_limit_per_prompt(self, modality):
+ del modality
+ return 3
+
+
+class _ModelConfig:
+ def __init__(self):
+ self.model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
+ self.revision = None
+ self.max_model_len = 4096
+ self.encoder_config = {}
+ self.dtype = torch.float32
+ self.hf_config = MossAudioConfig(language_config=Qwen3Config())
+ self.multimodal_config = _MMConfig()
+
+ def get_multimodal_config(self):
+ return self.multimodal_config
+
+ def get_inputs_embeds_size(self):
+ return None
+
+
+class _ProcessingContext:
+ def __init__(self):
+ self.model_config = _ModelConfig()
+ self.tokenizer = _Tokenizer()
+
+ def get_tokenizer(self):
+ return self.tokenizer
+
+ def get_hf_config(self):
+ return self.model_config.hf_config
+
+ def get_mm_config(self):
+ return self.model_config.get_multimodal_config()
+
+ def get_merged_mm_kwargs(self, kwargs):
+ return self.get_mm_config().merge_mm_processor_kwargs(kwargs)
+
+ def call_hf_processor(self, hf_processor, data, kwargs):
+ merged_kwargs = self.get_merged_mm_kwargs(kwargs)
+ merged_kwargs.setdefault("return_tensors", "pt")
+ return hf_processor(**data, **merged_kwargs)
+
+
+class _TestMossAudioProcessingInfo(MossAudioProcessingInfo):
+ def _get_processor_config_defaults(self):
+ return {}
+
+
+def _vllm_config(tensor_parallel_size=1, pipeline_parallel_size=1, hf_config=None):
+ if hf_config is None:
+ hf_config = MossAudioConfig(language_config=Qwen3Config())
+ return SimpleNamespace(
+ model_config=SimpleNamespace(
+ hf_config=hf_config,
+ multimodal_config=None,
+ ),
+ quant_config=None,
+ parallel_config=SimpleNamespace(
+ tensor_parallel_size=tensor_parallel_size,
+ pipeline_parallel_size=pipeline_parallel_size,
+ ),
+ )
+
+
+class _FakeAudioEncoder:
+ dtype = torch.float32
+
+ def __init__(self, deepstack_layers=0):
+ self.deepstack_layers = deepstack_layers
+ self.output_deepstack_hidden_states = None
+ self.input_shape = None
+ self.feature_lens = None
+
+ def __call__(self, audio_data, *, feature_lens, output_deepstack_hidden_states):
+ self.input_shape = tuple(audio_data.shape)
+ self.feature_lens = feature_lens.detach().cpu().clone()
+ self.output_deepstack_hidden_states = output_deepstack_hidden_states
+ lengths = MossAudioEncoder._compute_downsampled_length(feature_lens)
+ hidden_states = torch.ones(1, int(lengths.sum().item()), 8)
+ if not output_deepstack_hidden_states:
+ return hidden_states, None
+ return hidden_states, [
+ hidden_states * scale for scale in range(2, 2 + self.deepstack_layers)
+ ]
+
+
+def _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=1, tp_rank=0):
+ import vllm.model_executor.layers.linear as linear_layers
+ import vllm.model_executor.models.moss_audio as moss_audio_module
+ import vllm.model_executor.parameter as parameter_module
+
+ for module in (moss_audio_module, linear_layers, parameter_module):
+ monkeypatch.setattr(
+ module, "get_tensor_model_parallel_world_size", lambda: tp_size
+ )
+ monkeypatch.setattr(
+ linear_layers, "get_tensor_model_parallel_rank", lambda: tp_rank
+ )
+ monkeypatch.setattr(
+ parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
+ )
+ monkeypatch.setattr(
+ linear_layers, "tensor_model_parallel_all_reduce", lambda tensor: tensor
+ )
+
+
+def _build_moss_audio_processor(cache=None):
+ ctx = _ProcessingContext()
+ info = _TestMossAudioProcessingInfo(ctx)
+ return (
+ MossAudioMultiModalProcessor(
+ info,
+ MossAudioDummyInputsBuilder(info),
+ cache=cache,
+ ),
+ ctx,
+ )
+
+
+def _assert_mm_inputs_equal(left, right):
+ assert left["prompt_token_ids"] == right["prompt_token_ids"]
+ assert left["mm_hashes"] == right["mm_hashes"]
+
+ left_placeholder = left["mm_placeholders"]["audio"][0]
+ right_placeholder = right["mm_placeholders"]["audio"][0]
+ assert left_placeholder.offset == right_placeholder.offset
+ assert left_placeholder.length == right_placeholder.length
+ assert left_placeholder.is_embed.tolist() == right_placeholder.is_embed.tolist()
+
+ assert batched_tensors_equal(
+ left["mm_kwargs"].get_data(),
+ right["mm_kwargs"].get_data(),
+ )
+
+
+@pytest.mark.parametrize(
+ ("prompt", "prefix"),
+ [
+ (
+ f"before {MOSS_AUDIO_PLACEHOLDER} after",
+ [*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
+ ),
+ (
+ f"before {MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}"
+ f"{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN} after",
+ [*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
+ ),
+ ("Describe this audio.", [MOSS_AUDIO_BOS_TOKEN_ID]),
+ ],
+)
+def test_moss_audio_processor_expands_audio_placeholders(prompt, prefix):
+ raw_mel_len = 17
+ processed = MossAudioProcessor(_Tokenizer())(
+ text=prompt, audio=[torch.zeros(160 * raw_mel_len)]
+ )
+ input_ids = processed["input_ids"][0].tolist()
+
+ assert input_ids[: len(prefix)] == prefix
+ assert input_ids.count(MOSS_AUDIO_BOS_TOKEN_ID) == 1
+ assert input_ids.count(MOSS_AUDIO_EOS_TOKEN_ID) == 1
+ assert input_ids.count(MOSS_AUDIO_TOKEN_ID) == (
+ MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
+ )
+ assert processed["audio_data"].shape == (1, 128, raw_mel_len)
+ assert processed["audio_data_seqlens"].tolist() == [raw_mel_len]
+
+
+def test_moss_audio_processor_preserves_placeholder_without_audio():
+ processed = MossAudioProcessor(_Tokenizer())(
+ text=f"before {MOSS_AUDIO_PLACEHOLDER} after"
+ )
+
+ assert processed["input_ids"][0].tolist() == [
+ *[ord(char) for char in "before "],
+ MOSS_AUDIO_BOS_TOKEN_ID,
+ MOSS_AUDIO_TOKEN_ID,
+ MOSS_AUDIO_EOS_TOKEN_ID,
+ *[ord(char) for char in " after"],
+ ]
+ assert "audio_data" not in processed
+ assert "audio_data_seqlens" not in processed
+
+
+def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
+ raw_mel_len = 17
+ audio = np.zeros(160 * raw_mel_len, dtype=np.float32)
+ prompt = f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."
+
+ baseline_processor, ctx = _build_moss_audio_processor()
+ mm_items = baseline_processor.info.parse_mm_data({"audio": [audio]})
+ token_prompt = ctx.get_tokenizer().encode(prompt, add_special_tokens=False)
+
+ baseline_text = baseline_processor(
+ prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+ baseline_token = baseline_processor(
+ token_prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+
+ cache = MultiModalProcessorOnlyCache(ctx.model_config)
+ cached_processor, _ = _build_moss_audio_processor(cache=cache)
+ cached_text_miss = cached_processor(
+ prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+ cached_text_hit = cached_processor(
+ prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+ cached_token_hit = cached_processor(
+ token_prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+
+ expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
+ prompt_token_ids = baseline_text["prompt_token_ids"]
+ assert prompt_token_ids.count(MOSS_AUDIO_TOKEN_ID) == expected_audio_tokens
+ assert baseline_text["mm_placeholders"]["audio"][0].length == (
+ expected_audio_tokens + 2
+ )
+
+ _assert_mm_inputs_equal(baseline_text, baseline_token)
+ _assert_mm_inputs_equal(baseline_text, cached_text_miss)
+ _assert_mm_inputs_equal(baseline_text, cached_text_hit)
+ _assert_mm_inputs_equal(baseline_text, cached_token_hit)
+
+
+def test_moss_audio_supports_language_model_lora_only():
+ assert supports_lora(MossAudioModel)
+
+ model = object.__new__(MossAudioModel)
+ assert isinstance(model, SupportsLoRA)
+
+ mapping = model.get_mm_mapping()
+ assert mapping.language_model == ["language_model."]
+ assert mapping.tower_model == []
+ assert mapping.connector == []
+
+
+def test_moss_audio_error_paths():
+ model = object.__new__(MossAudioModel)
+ with pytest.raises(ValueError, match="DeepStack audio token count mismatch"):
+ model._cache_deepstack_input_embeds(
+ inputs_embeds=torch.zeros(4, 8),
+ deepstack_embeddings=((torch.ones(1, 8),),),
+ is_multimodal=torch.tensor([False, True, True, False]),
+ )
+
+ with pytest.raises(ValueError, match="too short"):
+ MossAudioProcessor(_Tokenizer())(
+ text=MOSS_AUDIO_PLACEHOLDER, audio=[torch.empty(0)]
+ )
+ with pytest.raises(ValueError, match="too short"):
+ model._parse_and_validate_audio_input(
+ audio_data=torch.zeros(1, 128, 1),
+ audio_data_seqlens=torch.tensor([0], dtype=torch.long),
+ )
+
+
+def test_moss_audio_validates_tp_config():
+ vllm_config = _vllm_config(tensor_parallel_size=2)
+ vllm_config.model_config.hf_config.adapter_hidden_size = 7
+
+ with pytest.raises(ValueError, match="adapter_hidden_size"):
+ MossAudioModel(vllm_config=vllm_config)
+
+ vllm_config = _vllm_config(tensor_parallel_size=2)
+ vllm_config.model_config.hf_config.audio_config.d_model = 6
+ vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3
+ with pytest.raises(ValueError, match="encoder_attention_heads"):
+ MossAudioModel(vllm_config=vllm_config)
+
+
+def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
+ model = object.__new__(MossAudioModel)
+
+ with pytest.raises(ValueError, match="audio_data batch size"):
+ model._parse_and_validate_audio_input(
+ audio_data=[torch.zeros(128, 8), torch.zeros(128, 11)],
+ audio_data_seqlens=torch.tensor([8], dtype=torch.long),
+ )
+
+
+@pytest.mark.parametrize("deepstack_scales", [(), (7, 11)])
+def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales):
+ model = object.__new__(MossAudioModel)
+ model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales))
+ model.audio_adapter = lambda hidden_states: hidden_states * 5
+ model.deepstack_audio_merger_list = [
+ lambda hidden_states, scale=scale: hidden_states * scale
+ for scale in deepstack_scales
+ ]
+ model.deepstack_input_embeds = None
+
+ embeddings = model.embed_multimodal(
+ audio_data=torch.zeros(2, 128, 9),
+ audio_data_seqlens=torch.tensor([8, 9], dtype=torch.long),
+ )
+
+ assert model.audio_encoder.output_deepstack_hidden_states is bool(deepstack_scales)
+ assert [embeds.shape for embeds in embeddings] == [
+ torch.Size([1, 8 * (1 + len(deepstack_scales))]),
+ torch.Size([2, 8 * (1 + len(deepstack_scales))]),
+ ]
+ if not deepstack_scales:
+ assert model.deepstack_input_embeds is None
+ return
+
+ main_embeddings, deepstack_embeddings = model._split_multimodal_embeddings(
+ embeddings, hidden_size=8
+ )
+ assert [embeds.shape for embeds in main_embeddings] == [
+ torch.Size([1, 8]),
+ torch.Size([2, 8]),
+ ]
+ assert [[e.shape for e in layer] for layer in deepstack_embeddings] == [
+ [torch.Size([1, 8]), torch.Size([2, 8])] for _ in deepstack_scales
+ ]
+ assert torch.equal(main_embeddings[0], torch.full((1, 8), 5.0))
+ for idx, scale in enumerate(deepstack_scales):
+ assert torch.equal(
+ deepstack_embeddings[idx][0],
+ torch.full((1, 8), float((idx + 2) * scale)),
+ )
+
+
+def test_moss_audio_embed_input_ids_caches_packed_deepstack():
+ class _FakeLanguageModel:
+ def embed_input_ids(self, input_ids):
+ return torch.zeros(input_ids.shape[0], 8)
+
+ model = object.__new__(MossAudioModel)
+ model.language_model = _FakeLanguageModel()
+ model.deepstack_audio_merger_list = [object(), object()]
+ model.deepstack_input_embeds = None
+ multimodal_embeddings = (
+ torch.cat([torch.full((1, 8), x) for x in (5.0, 14.0, 33.0)], dim=-1),
+ torch.cat([torch.full((2, 8), x) for x in (7.0, 22.0, 44.0)], dim=-1),
+ )
+ is_multimodal = torch.tensor([False, True, True, True, False])
+
+ inputs_embeds = model.embed_input_ids(
+ input_ids=torch.arange(5),
+ multimodal_embeddings=multimodal_embeddings,
+ is_multimodal=is_multimodal,
+ )
+
+ assert torch.equal(inputs_embeds[1], torch.full((8,), 5.0))
+ assert torch.equal(inputs_embeds[2], torch.full((8,), 7.0))
+ assert torch.equal(inputs_embeds[3], torch.full((8,), 7.0))
+ assert model.deepstack_input_embeds is not None
+ tensors = model.deepstack_input_embeds.tensors
+ assert set(tensors) == {"deepstack_input_embeds_0", "deepstack_input_embeds_1"}
+ for tensor in tensors.values():
+ assert tensor[is_multimodal].abs().sum() > 0
+ assert torch.equal(tensor[~is_multimodal], torch.zeros(2, 8))
+
+
+def _patch_pp_group(monkeypatch, *, first=True, last=True):
+ import vllm.model_executor.models.moss_audio as moss_audio_module
+
+ monkeypatch.setattr(
+ moss_audio_module,
+ "get_pp_group",
+ lambda: SimpleNamespace(is_first_rank=first, is_last_rank=last),
+ )
+
+
+def test_moss_audio_pp_forward_routes_deepstack(monkeypatch):
+ for first in (True, False):
+ calls: list[dict[str, object]] = []
+
+ def fake_lm(*args, _calls=calls, **kwargs):
+ del args
+ _calls.append(kwargs)
+ return torch.ones(1, 1)
+
+ _patch_pp_group(monkeypatch, first=first)
+ model = object.__new__(MossAudioModel)
+ torch.nn.Module.__init__(model)
+ model.language_model = fake_lm
+ cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)})
+ inter = IntermediateTensors(
+ {
+ "hidden_states": torch.ones(3, 8),
+ "residual": torch.zeros(3, 8),
+ "deepstack_input_embeds_0": torch.full((3, 8), 5.0),
+ }
+ )
+ inputs_embeds = torch.full((3, 8), 9.0)
+ model.deepstack_input_embeds = cached
+
+ model.forward(
+ input_ids=None,
+ positions=torch.arange(3),
+ intermediate_tensors=None if first else inter,
+ inputs_embeds=inputs_embeds if first else None,
+ )
+
+ kwargs = calls[0]
+ assert kwargs["inputs_embeds"] is (inputs_embeds if first else None)
+ assert kwargs["deepstack_input_embeds"] is (cached if first else inter)
+ assert model.deepstack_input_embeds is None
+
+ calls = []
+
+ def fake_lm_non_first_rank(*args, **kwargs):
+ del args
+ calls.append(kwargs)
+ return torch.ones(1, 1)
+
+ _patch_pp_group(monkeypatch, first=False)
+ model = object.__new__(MossAudioModel)
+ torch.nn.Module.__init__(model)
+ model.language_model = fake_lm_non_first_rank
+ model.deepstack_input_embeds = IntermediateTensors({})
+ inter = IntermediateTensors(
+ {
+ "hidden_states": torch.ones(3, 8),
+ "residual": torch.zeros(3, 8),
+ }
+ )
+
+ model.forward(
+ input_ids=None,
+ positions=torch.arange(3),
+ intermediate_tensors=inter,
+ inputs_embeds=torch.ones(3, 8),
+ )
+ assert calls[0]["inputs_embeds"] is None
+ assert calls[0]["deepstack_input_embeds"] is inter
+
+
+def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
+ class AddOne(torch.nn.Module):
+ def forward(self, positions, hidden_states, residual):
+ del positions, residual
+ return hidden_states + 1, torch.zeros_like(hidden_states)
+
+ def make_model(num_layers, deepstack_layers=None):
+ model = object.__new__(MossQwen3Model)
+ torch.nn.Module.__init__(model)
+ model.start_layer, model.end_layer = 0, num_layers
+ model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)])
+ model.norm = lambda hidden_states, residual: (hidden_states, residual)
+ model._maybe_add_hidden_state = lambda aux, *args: aux
+ model.deepstack_inject_layer_indices = (
+ range(0) if deepstack_layers is None else deepstack_layers
+ )
+ return model
+
+ _patch_pp_group(monkeypatch, first=True, last=True)
+ output = make_model(3).forward(
+ input_ids=None,
+ positions=torch.arange(2),
+ inputs_embeds=torch.zeros(2, 4),
+ deepstack_input_embeds=IntermediateTensors(
+ {
+ "deepstack_input_embeds_2": torch.full((2, 4), 5.0),
+ }
+ ),
+ )
+ assert torch.equal(output, torch.full((2, 4), 8.0))
+
+ _patch_pp_group(monkeypatch, first=True, last=False)
+ deepstack = IntermediateTensors(
+ {
+ "deepstack_input_embeds_0": torch.full((2, 4), 7.0),
+ "deepstack_input_embeds_3": torch.full((2, 4), 11.0),
+ }
+ )
+ output = make_model(2, range(4)).forward(
+ input_ids=None,
+ positions=torch.arange(2),
+ inputs_embeds=torch.zeros(2, 4),
+ deepstack_input_embeds=deepstack,
+ )
+ assert isinstance(output, IntermediateTensors)
+ assert set(output.tensors) == {
+ "hidden_states",
+ "residual",
+ "deepstack_input_embeds_2",
+ "deepstack_input_embeds_3",
+ }
+ assert torch.equal(output["hidden_states"], torch.full((2, 4), 9.0))
+ assert torch.equal(output["deepstack_input_embeds_2"], torch.zeros(2, 4))
+ assert output["deepstack_input_embeds_3"] is deepstack["deepstack_input_embeds_3"]
+
+ inner_model = make_model(0, range(2))
+ inner_model.make_empty_intermediate_tensors = lambda batch, dtype, device: (
+ IntermediateTensors(
+ {
+ "hidden_states": torch.zeros(batch, 4, dtype=dtype, device=device),
+ "residual": torch.zeros(batch, 4, dtype=dtype, device=device),
+ }
+ )
+ )
+ language_model = object.__new__(MossQwen3ForCausalLM)
+ torch.nn.Module.__init__(language_model)
+ language_model.model = inner_model
+ language_model.config = SimpleNamespace(hidden_size=4)
+ language_model.deepstack_inject_layer_indices = range(2)
+
+ tensors = MossQwen3ForCausalLM.make_empty_intermediate_tensors(
+ language_model,
+ batch_size=3,
+ dtype=torch.float16,
+ device=torch.device("cpu"),
+ )
+
+ assert set(tensors.tensors) == {
+ "hidden_states",
+ "residual",
+ "deepstack_input_embeds_0",
+ "deepstack_input_embeds_1",
+ }
+ assert tensors["deepstack_input_embeds_0"].shape == (3, 4)
+ assert tensors["deepstack_input_embeds_0"].dtype == torch.float16
+
+ _patch_pp_group(monkeypatch, first=True, last=False)
+ forward_tensors = inner_model.forward(
+ input_ids=None,
+ positions=torch.arange(3),
+ inputs_embeds=torch.ones(3, 4, dtype=torch.float16),
+ deepstack_input_embeds=None,
+ )
+ assert isinstance(forward_tensors, IntermediateTensors)
+ assert set(forward_tensors.tensors) == set(tensors.tensors)
+
+
+@pytest.mark.parametrize("tp_size", [1, 2])
+def test_moss_audio_gated_mlp_tp_shapes_and_loading(monkeypatch, tp_size):
+ from vllm.config import VllmConfig, set_current_vllm_config
+ from vllm.config.device import DeviceConfig
+
+ _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=tp_size)
+ with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
+ mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
+
+ params = dict(mlp.named_parameters())
+ assert params["gate_up_proj.weight"].shape == torch.Size([16 // tp_size, 4])
+ assert params["down_proj.weight"].shape == torch.Size([6, 8 // tp_size])
+
+ gate_weight = torch.arange(32, dtype=torch.float32).reshape(8, 4)
+ up_weight = torch.arange(100, 132, dtype=torch.float32).reshape(8, 4)
+ down_weight = torch.arange(48, dtype=torch.float32).reshape(6, 8)
+ loaded = mlp.load_weights(
+ [
+ ("gate_proj.weight", gate_weight),
+ ("up_proj.weight", up_weight),
+ ("down_proj.weight", down_weight),
+ ]
+ )
+
+ assert loaded == {"gate_up_proj.weight", "down_proj.weight"}
+ shard = 8 // tp_size
+ assert torch.equal(params["gate_up_proj.weight"][:shard], gate_weight[:shard])
+ assert torch.equal(params["gate_up_proj.weight"][shard:], up_weight[:shard])
+ assert torch.equal(params["down_proj.weight"], down_weight[:, : 8 // tp_size])
+
+ with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
+ packed_mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
+ packed_params = dict(packed_mlp.named_parameters())
+ loaded = packed_mlp.load_weights(
+ [("gate_up_proj.weight", torch.cat([gate_weight, up_weight], dim=0))]
+ )
+ assert loaded == {"gate_up_proj.weight"}
+ assert torch.equal(
+ packed_params["gate_up_proj.weight"][:shard],
+ gate_weight[:shard],
+ )
+ assert torch.equal(
+ packed_params["gate_up_proj.weight"][shard:],
+ up_weight[:shard],
+ )
+
+
+def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
+ from vllm.config import VllmConfig, set_current_vllm_config
+ from vllm.config.device import DeviceConfig
+
+ _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=2)
+ config = MossAudioEncoderConfig(
+ d_model=8,
+ output_dim=8,
+ num_mel_bins=8,
+ encoder_layers=1,
+ encoder_attention_heads=2,
+ encoder_ffn_dim=16,
+ downsample_hidden_size=2,
+ deepstack_encoder_layer_indexes=[],
+ )
+ with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
+ encoder = MossAudioEncoder(config)
+
+ attention = encoder.layers[0].self_attn
+ assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj"))
+ assert hasattr(attention, "out_proj")
+ assert not hasattr(attention, "qkv")
+ assert attention.k_proj.bias is None
+
+ weight_names = [
+ "layers.0.self_attn.q_proj.weight",
+ "layers.0.self_attn.q_proj.bias",
+ "layers.0.self_attn.k_proj.weight",
+ "layers.0.self_attn.v_proj.weight",
+ "layers.0.self_attn.v_proj.bias",
+ "layers.0.self_attn.out_proj.weight",
+ "layers.0.self_attn.out_proj.bias",
+ "conv1.weight",
+ "conv1.bias",
+ ]
+ params = dict(encoder.named_parameters(remove_duplicate=False))
+ assert "layers.0.self_attn.k_proj.bias" not in params
+ weights = {
+ name: torch.full_like(params[name], fill_value=float(i + 1))
+ for i, name in enumerate(weight_names)
+ }
+
+ loaded = AutoWeightsLoader(encoder).load_weights(weights.items())
+
+ assert "load_weights" not in MossAudioEncoder.__dict__
+ assert loaded == set(weight_names)
+ assert not any(".qkv." in name for name in loaded)
+ assert torch.equal(
+ params["layers.0.self_attn.q_proj.weight"],
+ weights["layers.0.self_attn.q_proj.weight"],
+ )
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 8f7ea82264..aa5971fb49 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1127,6 +1127,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {
tokenizer="moondream/starmie-v1",
trust_remote_code=True,
),
+ "MossAudioModel": _HfExamplesInfo(
+ "OpenMOSS-Team/MOSS-Audio-4B-Instruct",
+ extras={
+ "4b-thinking": "OpenMOSS-Team/MOSS-Audio-4B-Thinking",
+ "8b-instruct": "OpenMOSS-Team/MOSS-Audio-8B-Instruct",
+ "8b-thinking": "OpenMOSS-Team/MOSS-Audio-8B-Thinking",
+ },
+ trust_remote_code=True,
+ ),
"HfMoondream": _HfExamplesInfo(
"moondream/moondream3-preview",
tokenizer="moondream/starmie-v1",
diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py
index 2a8c81f55c..f6264f955f 100644
--- a/vllm/benchmarks/throughput.py
+++ b/vllm/benchmarks/throughput.py
@@ -21,6 +21,7 @@ from vllm.benchmarks.datasets import (
BenchmarkDataset,
BurstGPTDataset,
ConversationDataset,
+ CustomAudioDataset,
InstructCoderDataset,
MultiModalConversationDataset,
PrefixRepetitionRandomDataset,
@@ -257,9 +258,24 @@ def _run_vllm_chat_requests(
) -> tuple[float, list[RequestOutput]]:
from vllm import SamplingParams
- prompts = [request.prompt for request in requests]
+ prompts = []
sampling_params: list[SamplingParams] = []
for request in requests:
+ if isinstance(request.prompt, list):
+ prompts.append(request.prompt)
+ else:
+ content: list[dict[str, Any]] = [{"type": "text", "text": request.prompt}]
+ if request.multi_modal_data is not None:
+ if isinstance(request.multi_modal_data, list):
+ content.extend(request.multi_modal_data)
+ elif isinstance(request.multi_modal_data, dict):
+ content.append(request.multi_modal_data)
+ else:
+ raise TypeError(
+ "Could not process multimodal content of type: "
+ f"{type(request.multi_modal_data)}"
+ )
+ prompts.append([{"role": "user", "content": content}])
sampling_params.append(
SamplingParams(
n=n,
@@ -533,6 +549,7 @@ def get_requests(args, tokenizer):
"max_loras": args.max_loras,
"lora_assignment": getattr(args, "lora_assignment", "random"),
"num_requests": args.num_prompts,
+ "no_oversample": getattr(args, "no_oversample", False),
}
if args.dataset_name == "random" or (
@@ -573,6 +590,16 @@ def get_requests(args, tokenizer):
sample_kwargs["output_len"] = args.output_len
elif args.dataset_name == "burstgpt":
dataset_cls = BurstGPTDataset
+ elif args.dataset_name == "custom_audio":
+ dataset_cls = CustomAudioDataset
+ sample_kwargs["enable_multimodal_chat"] = getattr(
+ args, "enable_multimodal_chat", False
+ )
+ custom_output_len = getattr(args, "custom_output_len", None)
+ if custom_output_len is not None:
+ sample_kwargs["output_len"] = custom_output_len
+ elif args.output_len is not None:
+ sample_kwargs["output_len"] = args.output_len
elif args.dataset_name == "hf":
if args.output_len is not None:
sample_kwargs["output_len"] = args.output_len
@@ -917,6 +944,7 @@ def add_cli_args(parser: FlexibleArgumentParser):
"prefix_repetition",
"random-mm",
"random-rerank",
+ "custom_audio",
],
help="Name of the dataset to benchmark on.",
default="sharegpt",
@@ -933,6 +961,22 @@ def add_cli_args(parser: FlexibleArgumentParser):
parser.add_argument(
"--dataset-path", type=str, default=None, help="Path to the dataset"
)
+ parser.add_argument(
+ "--no-oversample",
+ action="store_true",
+ help="Do not oversample if the dataset has fewer samples than num-prompts.",
+ )
+ parser.add_argument(
+ "--enable-multimodal-chat",
+ action="store_true",
+ help="Enable multimodal chat transformation for datasets that support it.",
+ )
+ parser.add_argument(
+ "--custom-output-len",
+ type=int,
+ default=None,
+ help="Number of output tokens per request for custom datasets.",
+ )
parser.add_argument(
"--input-len",
type=int,
diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py
new file mode 100644
index 0000000000..00a7047c6c
--- /dev/null
+++ b/vllm/model_executor/models/moss_audio.py
@@ -0,0 +1,1892 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inference-only MOSS-Audio model compatible with HuggingFace weights."""
+
+import math
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Annotated, Any
+
+import numpy as np
+import regex as re
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers import BatchFeature, PretrainedConfig, Qwen3Config
+from transformers.models.whisper import WhisperFeatureExtractor
+
+from vllm.compilation.decorators import support_torch_compile
+from vllm.config import VllmConfig
+from vllm.config.multimodal import BaseDummyOptions
+from vllm.distributed import (
+ get_pp_group,
+ get_tensor_model_parallel_world_size,
+)
+from vllm.inputs import ModalityData, MultiModalDataDict
+from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY, SiluAndMul
+from vllm.model_executor.layers.linear import (
+ ColumnParallelLinear,
+ MergedColumnParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+ AudioItem,
+ MultiModalFieldConfig,
+ MultiModalKwargsItems,
+)
+from vllm.multimodal.parse import (
+ DictEmbeddingItems,
+ ModalityDataItems,
+ MultiModalDataItems,
+ MultiModalDataParser,
+)
+from vllm.multimodal.processing import (
+ BaseDummyInputsBuilder,
+ BaseMultiModalProcessor,
+ BaseProcessingInfo,
+ PromptReplacement,
+ PromptUpdate,
+ PromptUpdateDetails,
+)
+from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+from .interfaces import (
+ MultiModalEmbeddings,
+ SupportsLoRA,
+ SupportsMultiModal,
+ SupportsPP,
+ _require_is_multimodal,
+)
+from .module_mapping import MultiModelKeys
+from .qwen3 import Qwen3ForCausalLM, Qwen3Model
+from .utils import (
+ AutoWeightsLoader,
+ WeightsMapper,
+ _merge_multimodal_embeddings,
+ maybe_prefix,
+)
+
+MOSS_AUDIO_TOKEN = "<|AUDIO|>"
+MOSS_AUDIO_BOS_TOKEN = "<|audio_bos|>"
+MOSS_AUDIO_EOS_TOKEN = "<|audio_eos|>"
+MOSS_AUDIO_TOKEN_ID = 151654
+MOSS_AUDIO_BOS_TOKEN_ID = 151669
+MOSS_AUDIO_EOS_TOKEN_ID = 151670
+DEFAULT_MAX_AUDIO_SECONDS = 30
+DEFAULT_MOSS_AUDIO_MEL_CONFIG = {
+ "mel_dim": 128,
+ "mel_sr": 16000,
+ "mel_hop_length": 160,
+ "mel_n_fft": 400,
+}
+MOSS_AUDIO_PLACEHOLDER = (
+ f"{MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN}"
+)
+MOSS_AUDIO_SPAN_RE = re.compile(
+ f"{re.escape(MOSS_AUDIO_BOS_TOKEN)}"
+ f"(?:{re.escape(MOSS_AUDIO_TOKEN)})+"
+ f"{re.escape(MOSS_AUDIO_EOS_TOKEN)}"
+)
+MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {
+ "audio_token_id",
+ "audio_start_id",
+ "audio_end_id",
+ "enable_time_marker",
+ "mel_config",
+}
+
+
+class MossAudioAudioInputs(TensorSchema):
+ """
+ Dimensions:
+ - b: Batch size
+ - nmb: Number of mel bins
+ - t: Time frames
+ """
+
+ audio_data: Annotated[torch.Tensor, TensorShape("b", "nmb", "t")]
+ audio_data_seqlens: Annotated[torch.Tensor, TensorShape("b")]
+
+
+def _normalize_moss_audio_mel_config(
+ mel_config: Mapping[str, object] | None = None,
+) -> dict[str, int]:
+ config = dict(DEFAULT_MOSS_AUDIO_MEL_CONFIG)
+ config.update(_extract_moss_audio_mel_config(mel_config))
+ return config
+
+
+def _extract_moss_audio_mel_config(
+ mel_config: Mapping[str, object] | None = None,
+) -> dict[str, int]:
+ config: dict[str, int] = {}
+ if mel_config is None:
+ return config
+
+ aliases = {
+ "mel_dim": ("mel_dim", "feature_size", "n_mels", "num_mel_bins"),
+ "mel_sr": ("mel_sr", "sampling_rate", "sample_rate"),
+ "mel_hop_length": ("mel_hop_length", "hop_length"),
+ "mel_n_fft": ("mel_n_fft", "n_fft"),
+ }
+ for target_key, source_keys in aliases.items():
+ for source_key in source_keys:
+ if source_key in mel_config:
+ config[target_key] = int(mel_config[source_key])
+ break
+
+ return config
+
+
+def _filter_moss_audio_processor_config(
+ config: Mapping[str, object] | None,
+) -> dict[str, object]:
+ if not config:
+ return {}
+
+ return {
+ key: value
+ for key, value in config.items()
+ if key in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
+ }
+
+
+def _merge_moss_audio_processor_configs(
+ *configs: Mapping[str, object] | None,
+) -> dict[str, object]:
+ merged: dict[str, object] = {}
+ merged_mel_config: dict[str, int] = {}
+ for config in configs:
+ filtered = _filter_moss_audio_processor_config(config)
+ mel_config = filtered.pop("mel_config", None)
+ merged.update(filtered)
+ if isinstance(mel_config, Mapping):
+ merged_mel_config.update(_extract_moss_audio_mel_config(mel_config))
+
+ if merged_mel_config:
+ merged["mel_config"] = merged_mel_config
+ return merged
+
+
+@dataclass
+class MossAudioEncoderConfig:
+ d_model: int = 1280
+ output_dim: int = 1280
+ num_mel_bins: int = 128
+ encoder_layers: int = 32
+ encoder_attention_heads: int = 20
+ encoder_ffn_dim: int = 5120
+ downsample_rate: int = 8
+ downsample_hidden_size: int = 480
+ encoder_attention_window_size: int = 100
+ max_source_positions: int = 1500
+ dropout: float = 0.1
+ attention_dropout: float = 0.1
+ activation_dropout: float = 0.0
+ activation_function: str = "gelu"
+ layer_norm_eps: float = 1e-5
+ _attn_implementation: str = "eager"
+ pretrained_path: str = ""
+ n_window: int = 200
+ conv_chunksize: int = 64
+ deepstack_encoder_layer_indexes: list[int] = field(
+ default_factory=lambda: [8, 16, 24]
+ )
+
+ @classmethod
+ def from_config(cls, config: object) -> "MossAudioEncoderConfig":
+ if isinstance(config, cls):
+ return config
+ if isinstance(config, Mapping):
+ values = {
+ key: value
+ for key, value in config.items()
+ if key in cls.__dataclass_fields__
+ }
+ else:
+ values = {
+ key: getattr(config, key)
+ for key in cls.__dataclass_fields__
+ if hasattr(config, key)
+ }
+ return cls(**values)
+
+
+class MossAudioConfig(PretrainedConfig):
+ model_type = "moss_audio"
+ is_composition = True
+
+ def __init__(
+ self,
+ audio_config: Mapping[str, object] | MossAudioEncoderConfig | None = None,
+ language_config: Mapping[str, object] | Qwen3Config | None = None,
+ adapter_hidden_size: int = 8192,
+ ignore_index: int = -100,
+ deepstack_num_inject_layers: int | None = None,
+ **kwargs: object,
+ ) -> None:
+ self.audio_config = MossAudioEncoderConfig.from_config(audio_config or {})
+ if isinstance(language_config, Qwen3Config):
+ self.language_config = language_config
+ else:
+ self.language_config = Qwen3Config(**(language_config or {}))
+
+ self.adapter_hidden_size = adapter_hidden_size
+ self.ignore_index = ignore_index
+ self.deepstack_num_inject_layers = deepstack_num_inject_layers
+
+ for key in ("num_hidden_layers", "eos_token_id", "bos_token_id", "vocab_size"):
+ kwargs.setdefault(key, getattr(self.language_config, key, None))
+ kwargs.setdefault("tie_word_embeddings", False)
+ super().__init__(**kwargs)
+
+ for key in (
+ "hidden_size",
+ "num_attention_heads",
+ "num_key_value_heads",
+ "head_dim",
+ "max_position_embeddings",
+ "rms_norm_eps",
+ ):
+ if hasattr(self.language_config, key):
+ setattr(self, key, getattr(self.language_config, key))
+
+ def get_text_config(self, decoder: bool = False) -> Qwen3Config:
+ return self.language_config
+
+
+class SinusoidsPositionEmbedding(nn.Module):
+ def __init__(self, num_positions: int, embedding_dim: int) -> None:
+ super().__init__()
+ del num_positions # Kept for config compatibility.
+ max_timescale = 10000.0
+ log_timescale_increment = math.log(max_timescale) / (embedding_dim // 2 - 1)
+ inv_timescales = torch.exp(
+ -log_timescale_increment * torch.arange(embedding_dim // 2).float()
+ )
+ self.register_buffer("inv_timescales", inv_timescales, persistent=False)
+
+ def forward(self, seq_len: int, device: torch.device) -> torch.Tensor:
+ scaled_time = (
+ torch.arange(seq_len, device=device, dtype=self.inv_timescales.dtype)[
+ :, None
+ ]
+ * self.inv_timescales[None, :]
+ )
+ return torch.cat([scaled_time.sin(), scaled_time.cos()], dim=1).unsqueeze(0)
+
+
+class MossAudioAttention(nn.Module):
+ def __init__(
+ self,
+ config: MossAudioEncoderConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.embed_dim = config.d_model
+ self.num_heads = config.encoder_attention_heads
+ self.head_dim = self.embed_dim // self.num_heads
+ if self.head_dim * self.num_heads != self.embed_dim:
+ raise ValueError(
+ f"d_model ({self.embed_dim}) must be divisible by "
+ f"encoder_attention_heads ({self.num_heads})."
+ )
+
+ tp_size = get_tensor_model_parallel_world_size()
+ if self.num_heads % tp_size != 0:
+ raise ValueError(
+ "MOSS-Audio audio encoder attention heads must be divisible by "
+ f"tensor parallel size. Got {self.num_heads=} and {tp_size=}."
+ )
+ self.num_local_heads = self.num_heads // tp_size
+ # TODO: can use QKVParallelLinear
+ self.q_proj = ColumnParallelLinear(
+ input_size=self.embed_dim,
+ output_size=self.embed_dim,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.q_proj",
+ )
+ self.k_proj = ColumnParallelLinear(
+ input_size=self.embed_dim,
+ output_size=self.embed_dim,
+ bias=False,
+ quant_config=quant_config,
+ prefix=f"{prefix}.k_proj",
+ )
+ self.v_proj = ColumnParallelLinear(
+ input_size=self.embed_dim,
+ output_size=self.embed_dim,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.v_proj",
+ )
+ self.out_proj = RowParallelLinear(
+ input_size=self.embed_dim,
+ output_size=self.embed_dim,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.out_proj",
+ )
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ ) -> torch.Tensor:
+ batch_size, seq_len, _ = hidden_states.shape
+ q, _ = self.q_proj(hidden_states)
+ k, _ = self.k_proj(hidden_states)
+ v, _ = self.v_proj(hidden_states)
+ q = q.view(batch_size, seq_len, -1, self.head_dim).transpose(1, 2)
+ k = k.view(batch_size, seq_len, -1, self.head_dim).transpose(1, 2)
+ v = v.view(batch_size, seq_len, -1, self.head_dim).transpose(1, 2)
+ attn_output = F.scaled_dot_product_attention(
+ q,
+ k,
+ v,
+ attn_mask=attention_mask[:, None, None, :],
+ dropout_p=0.0,
+ scale=self.head_dim**-0.5,
+ )
+ output, _ = self.out_proj(
+ attn_output.transpose(1, 2).reshape(
+ batch_size,
+ seq_len,
+ -1,
+ )
+ )
+ return output
+
+
+class MossAudioEncoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: MossAudioEncoderConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.self_attn = MossAudioAttention(
+ config,
+ quant_config=quant_config,
+ prefix=f"{prefix}.self_attn",
+ )
+ self.self_attn_layer_norm = nn.LayerNorm(
+ config.d_model, eps=config.layer_norm_eps
+ )
+ self.activation_fn = _ACTIVATION_REGISTRY[config.activation_function]
+ self.activation_dropout = config.activation_dropout
+ self.dropout = config.dropout
+ self.fc1 = ColumnParallelLinear(
+ config.d_model,
+ config.encoder_ffn_dim,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.fc1",
+ )
+ self.fc2 = RowParallelLinear(
+ config.encoder_ffn_dim,
+ config.d_model,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.fc2",
+ )
+ self.final_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ ) -> torch.Tensor:
+ residual = hidden_states
+ hidden_states = self.self_attn_layer_norm(hidden_states)
+ hidden_states = self.self_attn(hidden_states, attention_mask)
+ hidden_states = residual + F.dropout(
+ hidden_states, p=self.dropout, training=self.training
+ )
+
+ residual = hidden_states
+ hidden_states = self.final_layer_norm(hidden_states)
+ hidden_states, _ = self.fc1(hidden_states)
+ hidden_states = self.activation_fn(hidden_states)
+ hidden_states = F.dropout(
+ hidden_states, p=self.activation_dropout, training=self.training
+ )
+ hidden_states, _ = self.fc2(hidden_states)
+ hidden_states = residual + F.dropout(
+ hidden_states, p=self.dropout, training=self.training
+ )
+ return hidden_states
+
+
+class MossAudioEncoder(nn.Module):
+ def __init__(
+ self,
+ config: MossAudioEncoderConfig,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.config = config
+ self.gelu = nn.GELU()
+ self.conv1 = nn.Conv2d(
+ 1,
+ config.downsample_hidden_size,
+ kernel_size=(3, 3),
+ stride=(2, 2),
+ padding=(1, 1),
+ )
+ self.conv2 = nn.Conv2d(
+ config.downsample_hidden_size,
+ config.downsample_hidden_size,
+ kernel_size=(3, 3),
+ stride=(2, 2),
+ padding=(1, 1),
+ )
+ self.conv3 = nn.Conv2d(
+ config.downsample_hidden_size,
+ config.downsample_hidden_size,
+ kernel_size=(3, 3),
+ stride=(2, 2),
+ padding=(1, 1),
+ )
+
+ conv_freq = self._compute_downsampled_length(
+ torch.tensor(config.num_mel_bins)
+ ).item()
+ self.stem_proj = ReplicatedLinear(
+ config.downsample_hidden_size * int(conv_freq),
+ config.d_model,
+ bias=True,
+ quant_config=quant_config,
+ return_bias=False,
+ prefix=f"{prefix}.stem_proj",
+ )
+ self.embed_positions = SinusoidsPositionEmbedding(
+ config.max_source_positions, config.d_model
+ )
+ self.layers = nn.ModuleList(
+ [
+ MossAudioEncoderLayer(
+ config,
+ quant_config=quant_config,
+ prefix=f"{prefix}.layers.{layer_idx}",
+ )
+ for layer_idx in range(config.encoder_layers)
+ ]
+ )
+ self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_eps)
+ if config.output_dim != config.d_model:
+ self.out_proj = ReplicatedLinear(
+ config.d_model,
+ config.output_dim,
+ bias=False,
+ quant_config=quant_config,
+ return_bias=False,
+ prefix=f"{prefix}.out_proj",
+ )
+ else:
+ self.out_proj = nn.Identity()
+
+ self.deepstack_encoder_layer_indexes = list(
+ config.deepstack_encoder_layer_indexes or []
+ )
+ self._deepstack_capture_map = {
+ layer_idx: capture_idx
+ for capture_idx, layer_idx in enumerate(
+ self.deepstack_encoder_layer_indexes
+ )
+ }
+ self.n_window = int(config.n_window)
+ self.chunk_frames = int(self.n_window * 2)
+ self.conv_chunksize = int(config.conv_chunksize)
+
+ @property
+ def dtype(self) -> torch.dtype:
+ return self.conv1.weight.dtype
+
+ @staticmethod
+ def _compute_downsampled_length(lengths: torch.Tensor) -> torch.Tensor:
+ def conv_out_len(length: torch.Tensor) -> torch.Tensor:
+ return (length - 1) // 2 + 1
+
+ return conv_out_len(conv_out_len(conv_out_len(lengths)))
+
+ @staticmethod
+ def compute_num_audio_tokens(raw_mel_len: int) -> int:
+ lengths = torch.tensor(raw_mel_len, dtype=torch.long)
+ return int(MossAudioEncoder._compute_downsampled_length(lengths).item())
+
+ def _encode_chunk_batch(
+ self,
+ input_features: torch.Tensor,
+ seq_lengths: torch.Tensor,
+ output_deepstack_hidden_states: bool = True,
+ ) -> tuple[torch.Tensor, list[torch.Tensor]]:
+ if input_features.dim() == 2:
+ input_features = input_features.unsqueeze(0)
+
+ downsampled_lengths = self._compute_downsampled_length(seq_lengths)
+ x = input_features.unsqueeze(1)
+ x = self.gelu(self.conv1(x))
+ x = self.gelu(self.conv2(x))
+ x = self.gelu(self.conv3(x))
+ x = x.permute(0, 3, 1, 2).contiguous().flatten(2)
+ x = self.stem_proj(x)
+
+ max_len = int(downsampled_lengths.max().item())
+ if x.size(1) > max_len:
+ x = x[:, :max_len, :]
+ x = x + self.embed_positions(x.shape[1], x.device).to(x.dtype)
+
+ attention_mask = (
+ torch.arange(x.size(1), device=x.device)[None, :]
+ < downsampled_lengths[:, None]
+ )
+
+ deepstack_hidden_states: list[torch.Tensor | None] = []
+ if output_deepstack_hidden_states:
+ deepstack_hidden_states = [None] * len(self.deepstack_encoder_layer_indexes)
+ for layer_idx, layer in enumerate(self.layers):
+ x = layer(x, attention_mask)
+ if output_deepstack_hidden_states:
+ capture_idx = self._deepstack_capture_map.get(layer_idx)
+ if capture_idx is not None:
+ deepstack_hidden_states[capture_idx] = x
+
+ x = self.layer_norm(x)
+ x = self.out_proj(x)
+
+ if not output_deepstack_hidden_states:
+ return x, []
+
+ ordered_deepstack_hidden_states = [
+ hidden_states
+ for hidden_states in deepstack_hidden_states
+ if hidden_states is not None
+ ]
+ ordered_deepstack_hidden_states = [
+ self.out_proj(hidden_states)
+ for hidden_states in ordered_deepstack_hidden_states
+ ]
+ return x, ordered_deepstack_hidden_states
+
+ def forward(
+ self,
+ input_features: torch.Tensor,
+ feature_lens: torch.Tensor | None = None,
+ output_deepstack_hidden_states: bool = True,
+ ) -> tuple[torch.Tensor, tuple[torch.Tensor, ...] | None]:
+ if input_features.dim() == 3:
+ if feature_lens is None:
+ feature_lens = torch.full(
+ (input_features.size(0),),
+ input_features.size(-1),
+ dtype=torch.long,
+ device=input_features.device,
+ )
+ else:
+ feature_lens = feature_lens.to(
+ device=input_features.device, dtype=torch.long
+ )
+ valid_chunks = [
+ input_features[i, :, : int(feature_lens[i].item())]
+ for i in range(int(input_features.shape[0]))
+ ]
+ input_features = torch.cat(valid_chunks, dim=1)
+ elif input_features.dim() != 2:
+ raise ValueError(
+ f"Expected [n_mels, T] or [B, n_mels, T], got "
+ f"{tuple(input_features.shape)}."
+ )
+
+ if feature_lens is None:
+ feature_lens = torch.tensor(
+ [int(input_features.shape[1])],
+ device=input_features.device,
+ dtype=torch.long,
+ )
+ else:
+ feature_lens = feature_lens.to(
+ device=input_features.device, dtype=torch.long
+ )
+
+ chunk_num = torch.ceil(
+ feature_lens.to(torch.float32) / self.chunk_frames
+ ).long()
+ chunk_lengths = torch.full(
+ (int(chunk_num.sum().item()),),
+ self.chunk_frames,
+ dtype=torch.long,
+ device=feature_lens.device,
+ )
+ tail_chunk_index = F.pad(chunk_num, (1, 0), value=-1).cumsum(0)[1:]
+ chunk_lengths[tail_chunk_index] = feature_lens % self.chunk_frames
+ chunk_lengths[chunk_lengths == 0] = self.chunk_frames
+
+ chunk_list = input_features.T.split(chunk_lengths.tolist(), dim=0)
+ padded_feature = nn.utils.rnn.pad_sequence(
+ chunk_list, batch_first=True
+ ).transpose(1, 2)
+
+ feature_lens_after_cnn = self._compute_downsampled_length(chunk_lengths)
+ t_down_max = (
+ int(feature_lens_after_cnn.max().item())
+ if feature_lens_after_cnn.numel() > 0
+ else 0
+ )
+ indices = torch.arange(t_down_max, device=padded_feature.device)
+ padded_mask_after_cnn = indices[None, :] < feature_lens_after_cnn[:, None]
+
+ num_deepstack = len(self.deepstack_encoder_layer_indexes)
+ should_output_deepstack = output_deepstack_hidden_states and num_deepstack > 0
+ padded_embeds: list[torch.Tensor] = []
+ deepstack_padded_embeds: list[list[torch.Tensor]] = [
+ [] for _ in range(num_deepstack if should_output_deepstack else 0)
+ ]
+ for feat_chunk, len_chunk in zip(
+ padded_feature.split(self.conv_chunksize, dim=0),
+ chunk_lengths.split(self.conv_chunksize, dim=0),
+ ):
+ out, deepstack_outs = self._encode_chunk_batch(
+ feat_chunk,
+ len_chunk,
+ output_deepstack_hidden_states=should_output_deepstack,
+ )
+ if out.shape[1] < t_down_max:
+ out = F.pad(out, (0, 0, 0, t_down_max - out.shape[1]))
+ padded_embeds.append(out)
+
+ if should_output_deepstack:
+ if len(deepstack_outs) != num_deepstack:
+ raise RuntimeError(
+ "DeepStack output count does not match configured "
+ "layer indexes."
+ )
+ for capture_idx, ds in enumerate(deepstack_outs):
+ if ds.shape[1] < t_down_max:
+ ds = F.pad(ds, (0, 0, 0, t_down_max - ds.shape[1]))
+ deepstack_padded_embeds[capture_idx].append(ds)
+
+ if padded_embeds:
+ padded_embed = torch.cat(padded_embeds, dim=0)
+ else:
+ padded_embed = torch.empty(
+ (0, t_down_max, self.config.output_dim),
+ device=padded_feature.device,
+ dtype=padded_feature.dtype,
+ )
+
+ last_hidden_state = padded_embed[padded_mask_after_cnn].unsqueeze(0)
+
+ deepstack_states: tuple[torch.Tensor, ...] | None = None
+ if should_output_deepstack:
+ collected: list[torch.Tensor] = []
+ for chunks_list in deepstack_padded_embeds:
+ if chunks_list:
+ ds = torch.cat(chunks_list, dim=0)
+ collected.append(ds[padded_mask_after_cnn].unsqueeze(0))
+ else:
+ collected.append(
+ torch.empty(
+ (1, 0, self.config.output_dim),
+ device=padded_feature.device,
+ dtype=padded_embed.dtype,
+ )
+ )
+ deepstack_states = tuple(collected)
+
+ return last_hidden_state, deepstack_states
+
+
+class GatedMLP(nn.Module):
+ def __init__(
+ self,
+ input_size: int,
+ hidden_size: int,
+ output_size: int,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.gate_up_proj = MergedColumnParallelLinear(
+ input_size,
+ [hidden_size, hidden_size],
+ bias=False,
+ quant_config=quant_config,
+ prefix=f"{prefix}.gate_up_proj",
+ )
+ self.down_proj = RowParallelLinear(
+ hidden_size,
+ output_size,
+ bias=False,
+ input_is_parallel=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.down_proj",
+ )
+ self.act_fn = SiluAndMul()
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ gate_up, _ = self.gate_up_proj(x)
+ x = self.act_fn(gate_up)
+ x, _ = self.down_proj(x)
+ return x
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ stacked_params_mapping = [
+ ("gate_up_proj", "gate_proj", 0),
+ ("gate_up_proj", "up_proj", 1),
+ ]
+ params_dict = dict(self.named_parameters())
+ loaded_params: set[str] = set()
+
+ for name, loaded_weight in weights:
+ target_name = name
+ for param_name, weight_name, shard_id in stacked_params_mapping:
+ components = target_name.split(".")
+ if weight_name not in components:
+ continue
+
+ target_name = ".".join(
+ param_name if component == weight_name else component
+ for component in components
+ )
+ param = params_dict[target_name]
+ weight_loader = param.weight_loader
+ weight_loader(param, loaded_weight, shard_id)
+ break
+ else:
+ param = params_dict[target_name]
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
+ weight_loader(param, loaded_weight)
+
+ loaded_params.add(target_name)
+ return loaded_params
+
+
+@support_torch_compile(
+ dynamic_arg_dims={
+ "input_ids": 0,
+ "positions": -1,
+ "intermediate_tensors": 0,
+ "inputs_embeds": 0,
+ "deepstack_input_embeds": 0,
+ }
+)
+class MossQwen3Model(Qwen3Model):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
+ self.deepstack_inject_layer_indices: Iterable[int] = range(0)
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ deepstack_input_embeds: IntermediateTensors | None = None,
+ ) -> torch.Tensor | IntermediateTensors:
+ if get_pp_group().is_first_rank:
+ if inputs_embeds is not None:
+ hidden_states = inputs_embeds
+ else:
+ hidden_states = self.embed_input_ids(input_ids)
+ residual = None
+ else:
+ assert intermediate_tensors is not None
+ hidden_states = intermediate_tensors["hidden_states"]
+ residual = intermediate_tensors["residual"]
+
+ aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
+ for layer_idx, layer in enumerate(
+ self.layers[self.start_layer : self.end_layer],
+ start=self.start_layer,
+ ):
+ hidden_states, residual = layer(positions, hidden_states, residual)
+ deepstack_key = f"deepstack_input_embeds_{layer_idx}"
+ if (
+ deepstack_input_embeds is not None
+ and deepstack_key in deepstack_input_embeds.tensors
+ ):
+ hidden_states = hidden_states + deepstack_input_embeds[deepstack_key]
+ self._maybe_add_hidden_state(
+ aux_hidden_states,
+ layer_idx - self.start_layer + 1,
+ hidden_states,
+ residual,
+ )
+
+ if not get_pp_group().is_last_rank:
+ tensors = {"hidden_states": hidden_states, "residual": residual}
+ # Keep the DeepStack PP schema config-driven, but only carry
+ # payloads needed by downstream injection points across this rank.
+ # Missing downstream payloads are zero-filled below to clear
+ # receive buffers instead of leaving stale tensors.
+ for layer_idx in self.deepstack_inject_layer_indices:
+ if layer_idx < self.end_layer:
+ continue
+ deepstack_key = f"deepstack_input_embeds_{layer_idx}"
+ if (
+ deepstack_input_embeds is not None
+ and deepstack_key in deepstack_input_embeds.tensors
+ ):
+ tensors[deepstack_key] = deepstack_input_embeds[deepstack_key]
+ else:
+ tensors[deepstack_key] = hidden_states.new_zeros(
+ hidden_states.shape
+ )
+ return IntermediateTensors(tensors)
+
+ hidden_states, _ = self.norm(hidden_states, residual)
+ if len(aux_hidden_states) > 0:
+ return hidden_states, aux_hidden_states
+ return hidden_states
+
+
+class MossQwen3ForCausalLM(Qwen3ForCausalLM):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+ super(Qwen3ForCausalLM, self).__init__()
+ config = vllm_config.model_config.hf_config
+ quant_config = vllm_config.quant_config
+
+ self.config = config
+ self.vllm_config = vllm_config
+ self.quant_config = quant_config
+ self.model = MossQwen3Model(
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
+ )
+
+ if get_pp_group().is_last_rank:
+ if config.tie_word_embeddings:
+ self.lm_head = self.model.embed_tokens
+ else:
+ self.lm_head = ParallelLMHead(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ prefix=maybe_prefix(prefix, "lm_head"),
+ )
+ else:
+ from .utils import PPMissingLayer
+
+ self.lm_head = PPMissingLayer()
+
+ self.logits_processor = LogitsProcessor(config.vocab_size)
+ self.deepstack_inject_layer_indices: Iterable[int] = range(0)
+
+ def make_empty_intermediate_tensors(
+ self,
+ batch_size: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ ) -> IntermediateTensors:
+ intermediate_tensors = self.model.make_empty_intermediate_tensors(
+ batch_size, dtype, device
+ )
+ for layer_idx in self.deepstack_inject_layer_indices:
+ intermediate_tensors[f"deepstack_input_embeds_{layer_idx}"] = torch.zeros(
+ (batch_size, self.config.hidden_size),
+ dtype=dtype,
+ device=device,
+ )
+ return intermediate_tensors
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ deepstack_input_embeds: IntermediateTensors | None = None,
+ ) -> torch.Tensor | IntermediateTensors:
+ return self.model(
+ input_ids,
+ positions,
+ intermediate_tensors,
+ inputs_embeds,
+ deepstack_input_embeds=deepstack_input_embeds,
+ )
+
+
+def _moss_audio_field_config(
+ hf_inputs: Mapping[str, torch.Tensor],
+) -> Mapping[str, MultiModalFieldConfig]:
+ return {
+ "audio_data": MultiModalFieldConfig.batched("audio"),
+ "audio_data_seqlens": MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
+ }
+
+
+class MossAudioMultiModalDataParser(MultiModalDataParser):
+ def _parse_audio_data(
+ self,
+ data: dict[str, torch.Tensor] | ModalityData[AudioItem],
+ ) -> ModalityDataItems[Any, Any] | None:
+ if isinstance(data, dict):
+ return DictEmbeddingItems(
+ data,
+ modality="audio",
+ required_fields={"audio_data", "audio_data_seqlens"},
+ fields_factory=_moss_audio_field_config,
+ )
+
+ return super()._parse_audio_data(data)
+
+
+class MossAudioProcessor:
+ model_input_names = [
+ "input_ids",
+ "attention_mask",
+ "audio_data",
+ "audio_data_seqlens",
+ ]
+
+ def __init__(
+ self,
+ tokenizer: object,
+ *,
+ audio_token_id: int = MOSS_AUDIO_TOKEN_ID,
+ audio_start_id: int = MOSS_AUDIO_BOS_TOKEN_ID,
+ audio_end_id: int = MOSS_AUDIO_EOS_TOKEN_ID,
+ enable_time_marker: bool = False,
+ mel_config: Mapping[str, object] | None = None,
+ ) -> None:
+ self.tokenizer = tokenizer
+ self.audio_token_id = int(audio_token_id)
+ self.audio_start_id = int(audio_start_id)
+ self.audio_end_id = int(audio_end_id)
+ self.enable_time_marker = bool(enable_time_marker)
+ self.mel_config = _normalize_moss_audio_mel_config(mel_config)
+ self.feature_extractor = WhisperFeatureExtractor(
+ feature_size=self.mel_config["mel_dim"],
+ sampling_rate=self.mel_config["mel_sr"],
+ hop_length=self.mel_config["mel_hop_length"],
+ n_fft=self.mel_config["mel_n_fft"],
+ )
+ self.audio_tokens_per_second = self.mel_config["mel_sr"] / (
+ self.mel_config["mel_hop_length"] * 8
+ )
+ self.time_marker_every_seconds = 2
+ self.time_marker_every_audio_tokens = int(
+ self.audio_tokens_per_second * self.time_marker_every_seconds
+ )
+ self._digit_token_ids = {
+ "0": 15,
+ "1": 16,
+ "2": 17,
+ "3": 18,
+ "4": 19,
+ "5": 20,
+ "6": 21,
+ "7": 22,
+ "8": 23,
+ "9": 24,
+ }
+
+ @staticmethod
+ def conv3_downsample_len(raw_mel_len: int) -> int:
+ return MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
+
+ def _extract_mel(self, audio: np.ndarray | torch.Tensor) -> torch.Tensor:
+ if isinstance(audio, torch.Tensor):
+ wav = audio.detach().to("cpu", dtype=torch.float32).numpy()
+ else:
+ wav = np.asarray(audio, dtype=np.float32)
+ if wav.size == 0:
+ raise ValueError("The audio is too short to be represented.")
+ if wav.ndim == 2:
+ wav = wav[0]
+ feats = self.feature_extractor._np_extract_fbank_features(
+ wav[None, ...], device="cpu"
+ )
+ return torch.from_numpy(feats[0])
+
+ def _get_default_audio_prompt(self) -> str:
+ return MOSS_AUDIO_PLACEHOLDER
+
+ def _ensure_audio_placeholders(
+ self,
+ prompt_text: str,
+ num_audios: int,
+ ) -> str:
+ if num_audios == 0 or MOSS_AUDIO_SPAN_RE.search(prompt_text):
+ return prompt_text
+
+ audio_prompt = self._get_default_audio_prompt() * num_audios
+ if prompt_text:
+ return f"{audio_prompt}\n{prompt_text}"
+ return audio_prompt
+
+ def _build_audio_tokens_with_time_markers(self, audio_seq_len: int) -> list[int]:
+ total_duration_seconds = audio_seq_len / self.audio_tokens_per_second
+ num_full_seconds = int(total_duration_seconds)
+ token_ids: list[int] = []
+ audio_tokens_consumed = 0
+ for second in range(
+ self.time_marker_every_seconds,
+ num_full_seconds + 1,
+ self.time_marker_every_seconds,
+ ):
+ marker_pos = (
+ second // self.time_marker_every_seconds
+ ) * self.time_marker_every_audio_tokens
+ audio_segment_len = marker_pos - audio_tokens_consumed
+ if audio_segment_len > 0:
+ token_ids.extend([self.audio_token_id] * audio_segment_len)
+ audio_tokens_consumed += audio_segment_len
+ token_ids.extend(self._digit_token_ids[digit] for digit in str(second))
+
+ remaining = audio_seq_len - audio_tokens_consumed
+ if remaining > 0:
+ token_ids.extend([self.audio_token_id] * remaining)
+ return token_ids
+
+ def build_audio_placeholder_ids(self, num_audio_tokens: int) -> list[int]:
+ if self.enable_time_marker:
+ return self._build_audio_tokens_with_time_markers(num_audio_tokens)
+ return [self.audio_token_id] * num_audio_tokens
+
+ def __call__(
+ self,
+ text: str | Sequence[str] | None = None,
+ audios: Sequence[np.ndarray | torch.Tensor] | None = None,
+ audio: Sequence[np.ndarray | torch.Tensor] | None = None,
+ return_tensors: str = "pt",
+ **kwargs: object,
+ ) -> BatchFeature:
+ """Build text tokens and audio tensors for one MossAudio prompt.
+
+ Example:
+ text="Describe this.", audio=[waveform]
+ -> input_ids contains audio_start, N audio tokens, audio_end
+ -> audio_data has shape [1, mel_dim, max_time]
+ -> mel_dim is the number of mel filter-bank bins, 128 by default
+ -> audio_data_seqlens stores the unpadded mel length
+ """
+ del kwargs
+
+ # Step 1. Normalize text input; this processor handles one prompt.
+ if isinstance(text, (list, tuple)):
+ if len(text) != 1:
+ raise ValueError(f"Expected text batch size 1, got {len(text)}")
+ prompt_text = text[0]
+ elif text is None:
+ prompt_text = ""
+ else:
+ prompt_text = text
+
+ # Step 2. Accept either `audios` or `audio` and normalize to a list.
+ audio_list = audios if audios is not None else audio
+ audio_list = [] if audio_list is None else list(audio_list)
+
+ # Step 3. Convert waveforms to [mel_dim, time] mel features and token
+ # counts. mel_dim is the number of mel filter-bank bins.
+ mels: list[torch.Tensor] = []
+ raw_lengths: list[int] = []
+ token_lens: list[int] = []
+ for one_audio in audio_list:
+ mel = self._extract_mel(one_audio)
+ raw_len = int(mel.shape[-1])
+ num_tokens = self.conv3_downsample_len(raw_len)
+ if raw_len <= 0 or num_tokens <= 0:
+ raise ValueError("The audio is too short to be represented.")
+ mels.append(mel)
+ raw_lengths.append(raw_len)
+ token_lens.append(num_tokens)
+
+ # Step 4. Pad variable-length mel features into a batch tensor.
+ if mels:
+ max_length = max(raw_lengths)
+ audio_batch = torch.zeros(
+ (len(mels), self.mel_config["mel_dim"], max_length),
+ dtype=torch.float32,
+ )
+ for index, mel in enumerate(mels):
+ audio_batch[index, :, : mel.shape[-1]] = mel
+ audio_data_seqlens = torch.tensor(raw_lengths, dtype=torch.long)
+ else:
+ audio_batch = None
+ audio_data_seqlens = None
+
+ # Step 5. Ensure each audio item has a placeholder span in the prompt.
+ prompt_text = self._ensure_audio_placeholders(prompt_text, len(audio_list))
+ input_ids = []
+ cursor = 0
+
+ # Step 6. Text-only path: tokenize and preserve placeholder spans.
+ if not audio_list:
+ for match in MOSS_AUDIO_SPAN_RE.finditer(prompt_text):
+ prefix = prompt_text[cursor : match.start()]
+ input_ids.extend(
+ self.tokenizer.encode(prefix, add_special_tokens=False)
+ )
+ input_ids.extend(
+ [self.audio_start_id, self.audio_token_id, self.audio_end_id]
+ )
+ cursor = match.end()
+ suffix = prompt_text[cursor:]
+ input_ids.extend(self.tokenizer.encode(suffix, add_special_tokens=False))
+ data: dict[str, torch.Tensor] = {
+ "input_ids": torch.tensor([input_ids], dtype=torch.long),
+ "attention_mask": torch.ones((1, len(input_ids)), dtype=torch.long),
+ }
+ return BatchFeature(data=data, tensor_type=return_tensors)
+
+ # Step 7. Audio path: expand each placeholder to its audio-token count.
+ span_iter = iter(MOSS_AUDIO_SPAN_RE.finditer(prompt_text))
+ for item_idx, _ in enumerate(audio_list):
+ match = next(span_iter, None)
+ if match is None:
+ raise ValueError(
+ "Audio placeholder count mismatch: expected one "
+ f"{MOSS_AUDIO_PLACEHOLDER!r} span per audio item."
+ )
+ prefix = prompt_text[cursor : match.start()]
+ input_ids.extend(self.tokenizer.encode(prefix, add_special_tokens=False))
+ input_ids.append(self.audio_start_id)
+ input_ids.extend(self.build_audio_placeholder_ids(token_lens[item_idx]))
+ input_ids.append(self.audio_end_id)
+ cursor = match.end()
+
+ # Step 8. Reject extra placeholder spans after all audio items are used.
+ suffix = prompt_text[cursor:]
+ if MOSS_AUDIO_SPAN_RE.search(suffix):
+ raise ValueError(
+ "Audio placeholder count mismatch: found more placeholder spans "
+ "than audio items."
+ )
+ input_ids.extend(self.tokenizer.encode(suffix, add_special_tokens=False))
+
+ # Step 9. Return tokenizer output plus audio tensors for embed_multimodal.
+ data = {
+ "input_ids": torch.tensor([input_ids], dtype=torch.long),
+ "attention_mask": torch.ones((1, len(input_ids)), dtype=torch.long),
+ }
+ if audio_batch is not None and audio_data_seqlens is not None:
+ data["audio_data"] = audio_batch
+ data["audio_data_seqlens"] = audio_data_seqlens
+ return BatchFeature(data=data, tensor_type=return_tensors)
+
+ def decode(self, *args: object, **kwargs: object) -> str:
+ return self.tokenizer.decode(*args, **kwargs)
+
+ def batch_decode(self, *args: object, **kwargs: object) -> list[str]:
+ return self.tokenizer.batch_decode(*args, **kwargs)
+
+
+class MossAudioProcessingInfo(BaseProcessingInfo):
+ def get_hf_config(self) -> MossAudioConfig:
+ config = self.ctx.get_hf_config()
+ if isinstance(config, MossAudioConfig):
+ return config
+ return MossAudioConfig(
+ audio_config=getattr(config, "audio_config", None),
+ language_config=getattr(config, "language_config", None),
+ adapter_hidden_size=getattr(config, "adapter_hidden_size", 8192),
+ ignore_index=getattr(config, "ignore_index", -100),
+ deepstack_num_inject_layers=getattr(
+ config, "deepstack_num_inject_layers", None
+ ),
+ )
+
+ def _get_processor_config_defaults(self) -> dict[str, object]:
+ cached_defaults = getattr(self, "_processor_config_defaults", None)
+ if cached_defaults is not None:
+ return cached_defaults
+
+ model_config = self.ctx.model_config
+ for file_name in ("processor_config.json", "preprocessor_config.json"):
+ config = get_hf_file_to_dict(
+ file_name,
+ model_config.model,
+ model_config.revision,
+ )
+ defaults = _filter_moss_audio_processor_config(config)
+ if defaults:
+ self._processor_config_defaults = defaults
+ return defaults
+
+ defaults = {}
+ self._processor_config_defaults = defaults
+ return defaults
+
+ @staticmethod
+ def _get_processor_cache_key(kwargs: Mapping[str, object]) -> tuple[object, ...]:
+ mel_config = _normalize_moss_audio_mel_config(
+ kwargs.get("mel_config")
+ if isinstance(kwargs.get("mel_config"), Mapping)
+ else None
+ )
+ return (
+ int(kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID)),
+ int(kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID)),
+ int(kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)),
+ bool(kwargs.get("enable_time_marker", False)),
+ tuple(sorted(mel_config.items())),
+ )
+
+ def get_hf_processor(self, **kwargs: object) -> MossAudioProcessor:
+ merged_kwargs = _merge_moss_audio_processor_configs(
+ self._get_processor_config_defaults(),
+ self.ctx.get_merged_mm_kwargs({}),
+ kwargs,
+ )
+ mel_config = _normalize_moss_audio_mel_config(
+ merged_kwargs.get("mel_config")
+ if isinstance(merged_kwargs.get("mel_config"), Mapping)
+ else None
+ )
+ processor_kwargs = {
+ "audio_token_id": int(
+ merged_kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID)
+ ),
+ "audio_start_id": int(
+ merged_kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID)
+ ),
+ "audio_end_id": int(
+ merged_kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)
+ ),
+ "enable_time_marker": bool(merged_kwargs.get("enable_time_marker", False)),
+ "mel_config": mel_config,
+ }
+
+ cache = getattr(self, "_hf_processor_cache", None)
+ if cache is None:
+ cache = {}
+ self._hf_processor_cache = cache
+
+ cache_key = self._get_processor_cache_key(processor_kwargs)
+ processor = cache.get(cache_key)
+ if processor is not None:
+ return processor
+
+ processor = MossAudioProcessor(
+ self.get_tokenizer(),
+ **processor_kwargs,
+ )
+ cache[cache_key] = processor
+ return processor
+
+ def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
+ return self.get_hf_processor(**kwargs).feature_extractor
+
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+ return {"audio": None}
+
+ def get_data_parser(self) -> MultiModalDataParser:
+ processor = self.get_hf_processor()
+ return MossAudioMultiModalDataParser(
+ target_sr=processor.mel_config["mel_sr"],
+ target_channels=1,
+ expected_hidden_size=self._get_expected_hidden_size(),
+ )
+
+ def get_mm_max_tokens_per_item(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ ) -> Mapping[str, int] | None:
+ if mm_counts.get("audio", 0) <= 0:
+ return {}
+ processor = self.get_hf_processor()
+ raw_mel_len = math.ceil(
+ (processor.mel_config["mel_sr"] * DEFAULT_MAX_AUDIO_SECONDS)
+ / processor.mel_config["mel_hop_length"]
+ )
+ return {"audio": MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)}
+
+
+class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo]):
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+ num_audios = mm_counts.get("audio", 0)
+ return MOSS_AUDIO_PLACEHOLDER * num_audios
+
+ def get_dummy_mm_data(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ mm_options: Mapping[str, BaseDummyOptions],
+ ) -> MultiModalDataDict:
+ num_audios = mm_counts.get("audio", 0)
+ audio_overrides = mm_options.get("audio")
+ return {
+ "audio": self._get_dummy_audios(
+ length=16000,
+ num_audios=num_audios,
+ overrides=audio_overrides,
+ )
+ }
+
+
+class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
+ def _call_hf_processor(
+ self,
+ prompt: str,
+ mm_data: Mapping[str, object],
+ mm_kwargs: Mapping[str, object],
+ tok_kwargs: Mapping[str, object],
+ ) -> BatchFeature:
+ mm_data = dict(mm_data)
+ audios = mm_data.pop("audios", [])
+ if audios:
+ mm_data["audio"] = audios
+ mm_kwargs = dict(mm_kwargs)
+ processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
+ tok_kwargs = {
+ key: value
+ for key, value in tok_kwargs.items()
+ if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
+ }
+ return self.info.ctx.call_hf_processor(
+ self.info.get_hf_processor(**processor_kwargs),
+ dict(text=prompt, **mm_data),
+ dict(**tok_kwargs),
+ )
+
+ def _get_mm_fields_config(
+ self,
+ hf_inputs: BatchFeature,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ ) -> Mapping[str, MultiModalFieldConfig]:
+ return _moss_audio_field_config(hf_inputs)
+
+ def _get_prompt_updates(
+ self,
+ mm_items: MultiModalDataItems,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ out_mm_kwargs: MultiModalKwargsItems,
+ ) -> Sequence[PromptUpdate]:
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+ out_mm_data = out_mm_kwargs.get_data()
+ audio_data_seqlens = out_mm_data.get("audio_data_seqlens")
+ if audio_data_seqlens is None:
+ audio_token_lens: list[int] = []
+ else:
+ if isinstance(audio_data_seqlens, torch.Tensor):
+ lens = audio_data_seqlens.reshape(-1).tolist()
+ else:
+ lens = list(audio_data_seqlens)
+ audio_token_lens = [
+ MossAudioEncoder.compute_num_audio_tokens(int(length))
+ for length in lens
+ ]
+
+ def get_replacement(
+ item_idx: int,
+ suffix_token_ids: list[int] | None = None,
+ ) -> PromptUpdateDetails[list[int]]:
+ num_tokens = audio_token_lens[item_idx]
+ if num_tokens == 0:
+ raise ValueError("The audio is too short to be represented.")
+ audio_token_ids = processor.build_audio_placeholder_ids(num_tokens)
+ suffix_token_ids = suffix_token_ids or []
+ is_embed = torch.tensor(
+ [token_id == processor.audio_token_id for token_id in audio_token_ids],
+ dtype=torch.bool,
+ )
+ return PromptUpdateDetails(
+ full=[
+ processor.audio_start_id,
+ *audio_token_ids,
+ processor.audio_end_id,
+ *suffix_token_ids,
+ ],
+ is_embed=lambda _tokenizer, _seq: torch.cat(
+ [
+ torch.tensor([False]),
+ is_embed,
+ torch.tensor([False]),
+ torch.zeros(len(suffix_token_ids), dtype=torch.bool),
+ ]
+ ),
+ )
+
+ prompt_update_specs = [
+ (
+ [
+ processor.audio_start_id,
+ processor.audio_token_id,
+ processor.audio_end_id,
+ ],
+ [],
+ )
+ ]
+ for suffix in ("", "\n"):
+ tokenizer_target = processor.tokenizer.encode(
+ MOSS_AUDIO_PLACEHOLDER + suffix,
+ add_special_tokens=False,
+ )
+ suffix_token_ids = processor.tokenizer.encode(
+ suffix,
+ add_special_tokens=False,
+ )
+ if any(target == tokenizer_target for target, _ in prompt_update_specs):
+ continue
+ prompt_update_specs.append((tokenizer_target, suffix_token_ids))
+
+ return [
+ PromptReplacement(
+ modality="audio",
+ target=target,
+ replacement=(
+ lambda item_idx, suffix_token_ids=suffix_token_ids: get_replacement(
+ item_idx,
+ suffix_token_ids,
+ )
+ ),
+ )
+ for target, suffix_token_ids in prompt_update_specs
+ ]
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+ MossAudioMultiModalProcessor,
+ info=MossAudioProcessingInfo,
+ dummy_inputs=MossAudioDummyInputsBuilder,
+)
+class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
+ packed_modules_mapping = {
+ "qkv_proj": [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ ],
+ "gate_up_proj": [
+ "gate_proj",
+ "up_proj",
+ ],
+ }
+
+ embedding_modules = {
+ "embed_tokens": "input_embeddings",
+ "lm_head": "output_embeddings",
+ }
+
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_prefix={
+ "lm_head.": "language_model.lm_head.",
+ "language_model.embed_tokens.": "language_model.model.embed_tokens.",
+ "language_model.layers.": "language_model.model.layers.",
+ "language_model.norm.": "language_model.model.norm.",
+ }
+ )
+
+ def get_mm_mapping(self) -> MultiModelKeys:
+ return MultiModelKeys.from_string_field(
+ language_model="language_model.",
+ )
+
+ @classmethod
+ def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+ if modality.startswith("audio"):
+ return MOSS_AUDIO_PLACEHOLDER
+ raise ValueError("Only audio modality is supported")
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+ super().__init__()
+ self.vllm_config = vllm_config
+ config = vllm_config.model_config.hf_config
+ if not isinstance(config, MossAudioConfig):
+ config = MossAudioConfig(
+ audio_config=getattr(config, "audio_config", None),
+ language_config=getattr(config, "language_config", None),
+ adapter_hidden_size=getattr(config, "adapter_hidden_size", 8192),
+ ignore_index=getattr(config, "ignore_index", -100),
+ deepstack_num_inject_layers=getattr(
+ config, "deepstack_num_inject_layers", None
+ ),
+ )
+ self.config = config
+ self.quant_config = vllm_config.quant_config
+ self.multimodal_config = vllm_config.model_config.multimodal_config
+
+ parallel_config = vllm_config.parallel_config
+ tp_size = parallel_config.tensor_parallel_size
+ if self.config.adapter_hidden_size % tp_size != 0:
+ raise ValueError(
+ "MOSS-Audio adapter_hidden_size must be divisible by tensor "
+ f"parallel size. Got adapter_hidden_size="
+ f"{self.config.adapter_hidden_size} and tensor_parallel_size="
+ f"{tp_size}."
+ )
+
+ audio_config = MossAudioEncoderConfig.from_config(self.config.audio_config)
+ if audio_config.encoder_attention_heads % tp_size != 0:
+ raise ValueError(
+ "MOSS-Audio encoder_attention_heads must be divisible by "
+ "tensor parallel size. Got encoder_attention_heads="
+ f"{audio_config.encoder_attention_heads} and "
+ f"tensor_parallel_size={tp_size}."
+ )
+ language_config = self.config.language_config
+ self.audio_token_id = MOSS_AUDIO_TOKEN_ID
+ self.deepstack_input_embeds: IntermediateTensors | None = None
+
+ with self._mark_tower_model(vllm_config, "audio"):
+ self.audio_encoder = MossAudioEncoder(
+ audio_config,
+ quant_config=self.quant_config,
+ prefix=maybe_prefix(prefix, "audio_encoder"),
+ )
+ self.audio_adapter = GatedMLP(
+ input_size=audio_config.output_dim,
+ hidden_size=self.config.adapter_hidden_size,
+ output_size=language_config.hidden_size,
+ quant_config=self.quant_config,
+ prefix=maybe_prefix(prefix, "audio_adapter"),
+ )
+
+ deepstack_k = len(audio_config.deepstack_encoder_layer_indexes or [])
+ if self.config.deepstack_num_inject_layers is not None:
+ deepstack_k = min(
+ deepstack_k,
+ int(self.config.deepstack_num_inject_layers),
+ )
+ self.deepstack_audio_merger_list = nn.ModuleList(
+ [
+ GatedMLP(
+ input_size=audio_config.output_dim,
+ hidden_size=self.config.adapter_hidden_size,
+ output_size=language_config.hidden_size,
+ quant_config=self.quant_config,
+ prefix=maybe_prefix(
+ prefix,
+ f"deepstack_audio_merger_list.{layer_idx}",
+ ),
+ )
+ for layer_idx in range(deepstack_k)
+ ]
+ )
+
+ with self._mark_language_model(vllm_config):
+ self.language_model = MossQwen3ForCausalLM(
+ vllm_config=vllm_config.with_hf_config(
+ language_config, architectures=["Qwen3ForCausalLM"]
+ ),
+ prefix=maybe_prefix(prefix, "language_model"),
+ )
+ self.language_model.deepstack_inject_layer_indices = range(deepstack_k)
+ self.language_model.model.deepstack_inject_layer_indices = range(
+ deepstack_k
+ )
+
+ self.make_empty_intermediate_tensors = (
+ self.language_model.make_empty_intermediate_tensors
+ )
+
+ @staticmethod
+ def _validate_audio_batch_size(
+ audio_batch_size: int, audio_data_seqlens: torch.Tensor
+ ) -> None:
+ if audio_batch_size != audio_data_seqlens.numel():
+ raise ValueError(
+ "audio_data batch size does not match audio_data_seqlens: "
+ f"{audio_batch_size} != {audio_data_seqlens.numel()}."
+ )
+
+ @staticmethod
+ def _pad_audio_data_list(
+ audio_data: list[torch.Tensor],
+ audio_data_seqlens: torch.Tensor,
+ ) -> torch.Tensor:
+ if len(audio_data) == 0:
+ raise ValueError("audio_data list must not be empty.")
+ MossAudioModel._validate_audio_batch_size(len(audio_data), audio_data_seqlens)
+
+ # pad_sequence needs every item to share the same trailing feature
+ # layout, so validate the mel-major audio tensors before transposing.
+ first = audio_data[0]
+ if not isinstance(first, torch.Tensor):
+ raise TypeError("audio_data list items must be torch.Tensor.")
+ if first.ndim != 2:
+ raise ValueError("audio_data list items must have shape [mel_dim, time].")
+
+ mel_dim = first.shape[0]
+ dtype = first.dtype
+ device = first.device
+ for item in audio_data[1:]:
+ if not isinstance(item, torch.Tensor):
+ raise TypeError("audio_data list items must be torch.Tensor.")
+ if item.ndim != 2:
+ raise ValueError(
+ "audio_data list items must have shape [mel_dim, time]."
+ )
+ if item.shape[0] != mel_dim:
+ raise ValueError("audio_data list items must have the same mel_dim.")
+ if item.dtype != dtype:
+ raise TypeError("audio_data list items must have the same dtype.")
+ if item.device != device:
+ raise ValueError("audio_data list items must be on the same device.")
+
+ # Each item arrives as [mel_dim, time]. pad_sequence pads along dim 1
+ # after converting to [time, mel_dim], then we restore [batch, mel, time].
+ time_major = [item.transpose(0, 1) for item in audio_data]
+ padded = torch.nn.utils.rnn.pad_sequence(time_major, batch_first=True)
+ return padded.transpose(1, 2).contiguous()
+
+ def _parse_and_validate_audio_input(
+ self, **kwargs: object
+ ) -> MossAudioAudioInputs | None:
+ """Normalize and validate model-side audio kwargs.
+
+ If audio_data is provided, this checks that audio_data_seqlens is also
+ present, flattens sequence lengths to a long tensor, pads list inputs
+ to [batch, mel_dim, time], validates batch-size/sequence-length
+ agreement, and rejects empty, non-positive, or downsampled-zero audio
+ lengths.
+ """
+ audio_data = kwargs.pop("audio_data", None)
+ audio_data_seqlens = kwargs.pop("audio_data_seqlens", None)
+ if audio_data is None:
+ return None
+ if audio_data_seqlens is None:
+ raise ValueError(
+ "audio_data_seqlens is required when audio_data is provided."
+ )
+ if not isinstance(audio_data_seqlens, torch.Tensor):
+ audio_data_seqlens = torch.tensor(audio_data_seqlens, dtype=torch.long)
+ audio_data_seqlens = audio_data_seqlens.to(dtype=torch.long).reshape(-1)
+
+ if isinstance(audio_data, list):
+ audio_data = self._pad_audio_data_list(audio_data, audio_data_seqlens)
+ elif isinstance(audio_data, torch.Tensor):
+ if audio_data.ndim == 3:
+ self._validate_audio_batch_size(audio_data.shape[0], audio_data_seqlens)
+ else:
+ raise TypeError("audio_data must be a torch.Tensor or list[torch.Tensor].")
+
+ audio_token_lens = MossAudioEncoder._compute_downsampled_length(
+ audio_data_seqlens
+ )
+ if (
+ audio_data_seqlens.numel() == 0
+ or torch.any(audio_data_seqlens <= 0).item()
+ or torch.any(audio_token_lens <= 0).item()
+ ):
+ raise ValueError("The audio is too short to be represented.")
+ return MossAudioAudioInputs(
+ audio_data=audio_data,
+ audio_data_seqlens=audio_data_seqlens,
+ )
+
+ def _process_audio_input(
+ self,
+ audio_input: MossAudioAudioInputs,
+ ) -> tuple[torch.Tensor, ...]:
+ """Run the audio encoder and return one embedding tensor per audio.
+
+ Example:
+ audio_data=[2, 128, 1200], audio_data_seqlens=[800, 1200]
+ -> returns (audio0_embeds, audio1_embeds), split by token length
+ -> DeepStack packs each item as [main, layer0, ...] on dim -1
+ """
+ audio_data = audio_input["audio_data"]
+ audio_data_seqlens = audio_input["audio_data_seqlens"]
+ last_hidden_state, deepstack = self.audio_encoder(
+ audio_data.to(self.audio_encoder.dtype),
+ feature_lens=audio_data_seqlens,
+ output_deepstack_hidden_states=len(self.deepstack_audio_merger_list) > 0,
+ )
+ audio_embeds = self.audio_adapter(last_hidden_state)
+ audio_lengths = MossAudioEncoder._compute_downsampled_length(
+ audio_data_seqlens.to(device=audio_embeds.device, dtype=torch.long)
+ ).tolist()
+ main_embeddings = tuple(audio_embeds.squeeze(0).split(audio_lengths, dim=0))
+
+ deepstack_embeddings: list[tuple[torch.Tensor, ...]] = []
+ if deepstack is not None:
+ if len(deepstack) < len(self.deepstack_audio_merger_list):
+ raise RuntimeError(
+ "DeepStack output count does not match configured audio "
+ "merger count."
+ )
+ for idx, hidden_states in enumerate(
+ deepstack[: len(self.deepstack_audio_merger_list)]
+ ):
+ ds_embeds = self.deepstack_audio_merger_list[idx](hidden_states)
+ deepstack_embeddings.append(
+ tuple(ds_embeds.squeeze(0).split(audio_lengths, dim=0))
+ )
+
+ if not deepstack_embeddings:
+ return main_embeddings
+
+ return tuple(
+ torch.cat(
+ [
+ main_embedding,
+ *(
+ layer_embeddings[item_idx]
+ for layer_embeddings in deepstack_embeddings
+ ),
+ ],
+ dim=-1,
+ )
+ for item_idx, main_embedding in enumerate(main_embeddings)
+ )
+
+ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
+ if audio_input is None:
+ return ()
+ return self._process_audio_input(audio_input)
+
+ def _split_multimodal_embeddings(
+ self,
+ multimodal_embeddings: MultiModalEmbeddings,
+ hidden_size: int,
+ ) -> tuple[tuple[torch.Tensor, ...], tuple[tuple[torch.Tensor, ...], ...]]:
+ """Unpack audio embeddings before merging them into token embeddings.
+
+ embed_input_ids calls this on the output of embed_multimodal. Plain
+ audio embeddings already have width hidden_size and are returned as the
+ main embeddings for _merge_multimodal_embeddings. When DeepStack is
+ enabled, _process_audio_input packs each audio item as
+ [main, layer0, layer1, ...] along the last dimension so the standard
+ multimodal path can carry a single embedding object. This method splits
+ that packed layout back into main embeddings plus per-layer DeepStack
+ embeddings, which _cache_deepstack_input_embeds scatters and forward
+ passes into MossQwen3Model for layer injection.
+ """
+ if isinstance(multimodal_embeddings, torch.Tensor):
+ embeddings = tuple(multimodal_embeddings.unbind(0))
+ else:
+ embeddings = tuple(multimodal_embeddings)
+
+ if len(embeddings) == 0:
+ return (), ()
+
+ deepstack_count = len(self.deepstack_audio_merger_list)
+ if all(embedding.shape[-1] == hidden_size for embedding in embeddings):
+ return embeddings, ()
+
+ packed_hidden_size = hidden_size * (deepstack_count + 1)
+ if deepstack_count == 0 or any(
+ embedding.shape[-1] != packed_hidden_size for embedding in embeddings
+ ):
+ got = [int(embedding.shape[-1]) for embedding in embeddings]
+ raise ValueError(
+ "MOSS-Audio multimodal embedding width mismatch: expected "
+ f"{hidden_size} or {packed_hidden_size}, got {got}."
+ )
+
+ split_by_item = [
+ torch.split(embedding, hidden_size, dim=-1) for embedding in embeddings
+ ]
+ main_embeddings = tuple(parts[0] for parts in split_by_item)
+ deepstack_embeddings = tuple(
+ tuple(parts[layer_idx + 1] for parts in split_by_item)
+ for layer_idx in range(deepstack_count)
+ )
+ return main_embeddings, deepstack_embeddings
+
+ def _cache_deepstack_input_embeds(
+ self,
+ inputs_embeds: torch.Tensor,
+ deepstack_embeddings: tuple[tuple[torch.Tensor, ...], ...],
+ is_multimodal: torch.Tensor,
+ ) -> None:
+ if len(deepstack_embeddings) == 0:
+ self.deepstack_input_embeds = None
+ return
+ flat_by_layer = [
+ torch.cat(layer_embeds, dim=0).to(
+ device=inputs_embeds.device, dtype=inputs_embeds.dtype
+ )
+ for layer_embeds in deepstack_embeddings
+ ]
+ num_mm_tokens = int(is_multimodal.sum().item())
+ if any(layer.shape[0] != num_mm_tokens for layer in flat_by_layer):
+ got = [int(layer.shape[0]) for layer in flat_by_layer]
+ raise ValueError(
+ "DeepStack audio token count mismatch: "
+ f"expected {num_mm_tokens}, got {got}."
+ )
+ data = {}
+ for layer_idx, layer_embeds in enumerate(flat_by_layer):
+ scattered = inputs_embeds.new_zeros(inputs_embeds.shape)
+ scattered[is_multimodal] = layer_embeds
+ data[f"deepstack_input_embeds_{layer_idx}"] = scattered
+ self.deepstack_input_embeds = IntermediateTensors(data)
+
+ def embed_input_ids(
+ self,
+ input_ids: torch.Tensor,
+ multimodal_embeddings: MultiModalEmbeddings | None = None,
+ *,
+ is_multimodal: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ inputs_embeds = self._embed_text_input_ids(
+ input_ids,
+ self.language_model.embed_input_ids,
+ is_multimodal=is_multimodal,
+ )
+
+ self.deepstack_input_embeds = None
+ if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
+ return inputs_embeds
+ is_multimodal = _require_is_multimodal(is_multimodal)
+ multimodal_embeddings, deepstack_embeddings = self._split_multimodal_embeddings(
+ multimodal_embeddings,
+ hidden_size=int(inputs_embeds.shape[-1]),
+ )
+
+ inputs_embeds = _merge_multimodal_embeddings(
+ inputs_embeds=inputs_embeds,
+ multimodal_embeddings=multimodal_embeddings,
+ is_multimodal=is_multimodal,
+ )
+ self._cache_deepstack_input_embeds(
+ inputs_embeds,
+ deepstack_embeddings,
+ is_multimodal,
+ )
+ return inputs_embeds
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ **kwargs: object,
+ ) -> torch.Tensor | IntermediateTensors:
+ if intermediate_tensors is None:
+ deepstack_input_embeds = self.deepstack_input_embeds
+ else:
+ # Non-first PP ranks consume hidden states from intermediate_tensors.
+ # The executor may still pass dummy inputs_embeds during profiling.
+ inputs_embeds = None
+ deepstack_input_embeds = intermediate_tensors
+ hidden_states = self.language_model(
+ input_ids,
+ positions,
+ intermediate_tensors=intermediate_tensors,
+ inputs_embeds=inputs_embeds,
+ deepstack_input_embeds=deepstack_input_embeds,
+ )
+ self.deepstack_input_embeds = None
+ return hidden_states
+
+ def compute_logits(
+ self,
+ hidden_states: torch.Tensor,
+ ) -> torch.Tensor | None:
+ return self.language_model.compute_logits(hidden_states)
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(
+ self,
+ skip_prefixes=["audio_encoder.embed_positions"],
+ )
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 5fb28b1c76..a18d54acdc 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -500,6 +500,7 @@ _MULTIMODAL_MODELS = {
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
"Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"),
+ "MossAudioModel": ("moss_audio", "MossAudioModel"),
"HfMoondream": ("moondream3", "Moondream3ForCausalLM"),
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py
index f8a30748b9..483f1d8be8 100644
--- a/vllm/transformers_utils/model_arch_config_convertor.py
+++ b/vllm/transformers_utils/model_arch_config_convertor.py
@@ -580,6 +580,50 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
return max(head_dim, global_head_dim) or super().get_head_size()
+class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase):
+ def _language_config(self) -> PretrainedConfig:
+ return self.hf_config.language_config
+
+ def get_num_hidden_layers(self) -> int:
+ return getattr(self._language_config(), "num_hidden_layers", 0)
+
+ def get_total_num_attention_heads(self) -> int:
+ return getattr(self._language_config(), "num_attention_heads", 0)
+
+ def get_vocab_size(self) -> int:
+ return getattr(self._language_config(), "vocab_size", 0)
+
+ def get_hidden_size(self) -> int:
+ return getattr(self._language_config(), "hidden_size", 0)
+
+ def get_head_size(self) -> int:
+ head_dim = getattr(self._language_config(), "head_dim", None)
+ if head_dim is not None:
+ return head_dim
+ total_num_attention_heads = self.get_total_num_attention_heads()
+ if total_num_attention_heads == 0:
+ return 0
+ return self.get_hidden_size() // total_num_attention_heads
+
+ def get_total_num_kv_heads(self) -> int:
+ return getattr(
+ self._language_config(),
+ "num_key_value_heads",
+ self.get_total_num_attention_heads(),
+ )
+
+ def derive_max_model_len_and_key(self) -> tuple[float, str | None]:
+ language_config = self._language_config()
+ max_position_embeddings = getattr(
+ language_config,
+ "max_position_embeddings",
+ None,
+ )
+ if max_position_embeddings is None:
+ return super().derive_max_model_len_and_key()
+ return max_position_embeddings, "language_config.max_position_embeddings"
+
+
# hf_config.model_type -> convertor class
MODEL_ARCH_CONFIG_CONVERTORS = {
"cohere_asr": CohereAsrModelArchConfigConvertor,
@@ -604,6 +648,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"mimo_v2_flash": MimoV2ModelArchConfigConvertor,
"mimo_v2_mtp": MimoV2MTPModelArchConfigConvertor,
"mimo_v2_omni_mtp": MimoV2MTPModelArchConfigConvertor,
+ "moss_audio": MossAudioModelArchConfigConvertor,
"mpt": MPTModelArchConfigConvertor,
"nemotron-nas": NemotronNasModelArchConfigConvertor,
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,