diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 19cccdc12f..c0a034bd96 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -569,6 +569,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `GlmOcrForConditionalGeneration` | GLM-OCR | T + IE+ | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ | | `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + IE+ | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ | | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | +| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ | | `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | | | `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | | | `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ | @@ -709,6 +710,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition. | `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | | `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ | | `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ | +| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ | | `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ | | `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ | | `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/multimodal/generation/test_granite_speech.py b/tests/models/multimodal/generation/test_granite_speech.py index 038a15d057..3019f5f22d 100644 --- a/tests/models/multimodal/generation/test_granite_speech.py +++ b/tests/models/multimodal/generation/test_granite_speech.py @@ -30,11 +30,14 @@ def vllm_to_hf_output( MODEL_NAME = "ibm-granite/granite-speech-3.3-2b" MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech" +# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration). +MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus" # Audio lora co-exists directly in the 3.3 model directory, -# the 4.0 model has adapters merged into the weights. +# the 4.0 and 4.1-plus models have adapters merged into the weights. models: dict[str, str | None] = { MODEL_NAME: MODEL_NAME, MODEL_NAME_4_0: None, + MODEL_NAME_4_1_PLUS: None, } diff --git a/tests/models/registry.py b/tests/models/registry.py index 298ba63d01..576e3086d4 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -938,6 +938,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "ibm-granite/granite-speech-3.3-2b", extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"}, ), + "GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo( + "ibm-granite/granite-speech-4.1-2b-plus", + min_transformers_version="5.8.0", + ), "GLM4VForCausalLM": _HfExamplesInfo( "zai-org/glm-4v-9b", trust_remote_code=True, diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 5b4959dc20..5f97f8b9a5 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -614,8 +614,7 @@ class GraniteSpeechForConditionalGeneration( ) with self._mark_tower_model(vllm_config, "audio"): - # Conformer encoder - self.encoder = GraniteSpeechCTCEncoder( + self.encoder = self._build_encoder( config=config.encoder_config, quant_config=quant_config, prefix=maybe_prefix(prefix, "encoder"), @@ -633,6 +632,18 @@ class GraniteSpeechForConditionalGeneration( self.language_model.make_empty_intermediate_tensors ) + def _build_encoder( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> "GraniteSpeechCTCEncoder": + return GraniteSpeechCTCEncoder( + config=config, + quant_config=quant_config, + prefix=prefix, + ) + def _parse_and_validate_audio_input( self, **kwargs: object, diff --git a/vllm/model_executor/models/granite_speech_plus.py b/vllm/model_executor/models/granite_speech_plus.py new file mode 100644 index 0000000000..ba95cfed13 --- /dev/null +++ b/vllm/model_executor/models/granite_speech_plus.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only IBM Granite Speech Plus model.""" + +import torch +from transformers import PretrainedConfig + +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.multimodal import MULTIMODAL_REGISTRY + +from .granite_speech import ( + GraniteSpeechCTCEncoder, + GraniteSpeechDummyInputsBuilder, + GraniteSpeechForConditionalGeneration, + GraniteSpeechMultiModalProcessingInfo, + GraniteSpeechMultiModalProcessor, +) + +ISO639_1_SUPPORTED_LANGS = { + "en": "English", + "fr": "French", + "de": "German", + "pt": "Portuguese", + "es": "Spanish", +} + + +class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder): + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.input_linear(hidden_states) + # cat_hidden_layers selects non-negative layer indices (0 = encoder + # input, N = output of layer N) whose hidden states are concatenated + # along the feature dim *in addition to* the final hidden states, + # which are always appended last. + cat_layers = set(self.config.cat_hidden_layers or []) + exported_hidden_states = [] + + if 0 in cat_layers: + exported_hidden_states.append(hidden_states) + + for idx, layer in enumerate(self.layers, start=1): + hidden_states = layer(hidden_states, attention_dists=self.attention_dists) + + # Skip the final layer here since its output is always appended + # below; capturing it twice would double-append. + if idx in cat_layers and idx != self.num_layers: + exported_hidden_states.append(hidden_states) + + if idx == self.num_layers // 2: + hidden_states_mid = hidden_states.clone() + hidden_states_mid, _ = self.out(hidden_states_mid) + hidden_states_mid = self.softmax(hidden_states_mid) + hidden_states_mid, _ = self.out_mid(hidden_states_mid) + hidden_states += hidden_states_mid + + if exported_hidden_states: + hidden_states = torch.cat([*exported_hidden_states, hidden_states], dim=-1) + return hidden_states + + +@MULTIMODAL_REGISTRY.register_processor( + GraniteSpeechMultiModalProcessor, + info=GraniteSpeechMultiModalProcessingInfo, + dummy_inputs=GraniteSpeechDummyInputsBuilder, +) +class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration): + supported_languages = ISO639_1_SUPPORTED_LANGS + + def _build_encoder( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> GraniteSpeechCTCEncoder: + return GraniteSpeechPlusCTCEncoder( + config=config, + quant_config=quant_config, + prefix=prefix, + ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 3028f9257b..3fbebaf7dc 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -419,6 +419,10 @@ _MULTIMODAL_MODELS = { "granite_speech", "GraniteSpeechForConditionalGeneration", ), + "GraniteSpeechPlusForConditionalGeneration": ( + "granite_speech_plus", + "GraniteSpeechPlusForConditionalGeneration", + ), "Granite4VisionForConditionalGeneration": ( "granite4_vision", "Granite4VisionForConditionalGeneration",