Add model support for granite speech plus (#43519)

Signed-off-by: Zvi Kons[WSL] <zvi@il.ibm.com>
Signed-off-by: Zvi Kons (BlueVela) <zvi@il.ibm.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Zvi Kons
2026-06-04 14:47:48 +00:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 06ee2d8433
commit b21443e23c
6 changed files with 106 additions and 3 deletions
+2
View File
@@ -569,6 +569,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
| `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + I<sup>E+</sup> | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ |
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ |
@@ -709,6 +710,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ |
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
@@ -30,11 +30,14 @@ def vllm_to_hf_output(
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration).
MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus"
# Audio lora co-exists directly in the 3.3 model directory,
# the 4.0 model has adapters merged into the weights.
# the 4.0 and 4.1-plus models have adapters merged into the weights.
models: dict[str, str | None] = {
MODEL_NAME: MODEL_NAME,
MODEL_NAME_4_0: None,
MODEL_NAME_4_1_PLUS: None,
}
+4
View File
@@ -938,6 +938,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"ibm-granite/granite-speech-3.3-2b",
extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
),
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-4.1-2b-plus",
min_transformers_version="5.8.0",
),
"GLM4VForCausalLM": _HfExamplesInfo(
"zai-org/glm-4v-9b",
trust_remote_code=True,
+13 -2
View File
@@ -614,8 +614,7 @@ class GraniteSpeechForConditionalGeneration(
)
with self._mark_tower_model(vllm_config, "audio"):
# Conformer encoder
self.encoder = GraniteSpeechCTCEncoder(
self.encoder = self._build_encoder(
config=config.encoder_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "encoder"),
@@ -633,6 +632,18 @@ class GraniteSpeechForConditionalGeneration(
self.language_model.make_empty_intermediate_tensors
)
def _build_encoder(
self,
config: PretrainedConfig,
quant_config: QuantizationConfig | None,
prefix: str,
) -> "GraniteSpeechCTCEncoder":
return GraniteSpeechCTCEncoder(
config=config,
quant_config=quant_config,
prefix=prefix,
)
def _parse_and_validate_audio_input(
self,
**kwargs: object,
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only IBM Granite Speech Plus model."""
import torch
from transformers import PretrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from .granite_speech import (
GraniteSpeechCTCEncoder,
GraniteSpeechDummyInputsBuilder,
GraniteSpeechForConditionalGeneration,
GraniteSpeechMultiModalProcessingInfo,
GraniteSpeechMultiModalProcessor,
)
ISO639_1_SUPPORTED_LANGS = {
"en": "English",
"fr": "French",
"de": "German",
"pt": "Portuguese",
"es": "Spanish",
}
class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = self.input_linear(hidden_states)
# cat_hidden_layers selects non-negative layer indices (0 = encoder
# input, N = output of layer N) whose hidden states are concatenated
# along the feature dim *in addition to* the final hidden states,
# which are always appended last.
cat_layers = set(self.config.cat_hidden_layers or [])
exported_hidden_states = []
if 0 in cat_layers:
exported_hidden_states.append(hidden_states)
for idx, layer in enumerate(self.layers, start=1):
hidden_states = layer(hidden_states, attention_dists=self.attention_dists)
# Skip the final layer here since its output is always appended
# below; capturing it twice would double-append.
if idx in cat_layers and idx != self.num_layers:
exported_hidden_states.append(hidden_states)
if idx == self.num_layers // 2:
hidden_states_mid = hidden_states.clone()
hidden_states_mid, _ = self.out(hidden_states_mid)
hidden_states_mid = self.softmax(hidden_states_mid)
hidden_states_mid, _ = self.out_mid(hidden_states_mid)
hidden_states += hidden_states_mid
if exported_hidden_states:
hidden_states = torch.cat([*exported_hidden_states, hidden_states], dim=-1)
return hidden_states
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
supported_languages = ISO639_1_SUPPORTED_LANGS
def _build_encoder(
self,
config: PretrainedConfig,
quant_config: QuantizationConfig | None,
prefix: str,
) -> GraniteSpeechCTCEncoder:
return GraniteSpeechPlusCTCEncoder(
config=config,
quant_config=quant_config,
prefix=prefix,
)
+4
View File
@@ -419,6 +419,10 @@ _MULTIMODAL_MODELS = {
"granite_speech",
"GraniteSpeechForConditionalGeneration",
),
"GraniteSpeechPlusForConditionalGeneration": (
"granite_speech_plus",
"GraniteSpeechPlusForConditionalGeneration",
),
"Granite4VisionForConditionalGeneration": (
"granite4_vision",
"Granite4VisionForConditionalGeneration",