Add model support for granite speech plus (#43519)
Signed-off-by: Zvi Kons[WSL] <zvi@il.ibm.com> Signed-off-by: Zvi Kons (BlueVela) <zvi@il.ibm.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
06ee2d8433
commit
b21443e23c
@@ -569,6 +569,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
|
||||
| `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + I<sup>E+</sup> | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ |
|
||||
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
|
||||
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
|
||||
| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
|
||||
| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I<sup>+</sup> + V<sup>+</sup> | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
|
||||
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ |
|
||||
@@ -709,6 +710,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
|
||||
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
|
||||
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
|
||||
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-4.0-1b-speech`, `ibm-granite/granite-speech-3.3-2b`, etc. | ✅︎ | ✅︎ |
|
||||
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
|
||||
| `Qwen3ASRForConditionalGeneration` | Qwen3-ASR | `Qwen/Qwen3-ASR-1.7B`, etc. | ✅︎ | ✅︎ |
|
||||
| `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, etc. | | ✅︎ |
|
||||
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -30,11 +30,14 @@ def vllm_to_hf_output(
|
||||
|
||||
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
|
||||
MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
|
||||
# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration).
|
||||
MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus"
|
||||
# Audio lora co-exists directly in the 3.3 model directory,
|
||||
# the 4.0 model has adapters merged into the weights.
|
||||
# the 4.0 and 4.1-plus models have adapters merged into the weights.
|
||||
models: dict[str, str | None] = {
|
||||
MODEL_NAME: MODEL_NAME,
|
||||
MODEL_NAME_4_0: None,
|
||||
MODEL_NAME_4_1_PLUS: None,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -938,6 +938,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"ibm-granite/granite-speech-3.3-2b",
|
||||
extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"},
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-speech-4.1-2b-plus",
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"GLM4VForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/glm-4v-9b",
|
||||
trust_remote_code=True,
|
||||
|
||||
@@ -614,8 +614,7 @@ class GraniteSpeechForConditionalGeneration(
|
||||
)
|
||||
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
# Conformer encoder
|
||||
self.encoder = GraniteSpeechCTCEncoder(
|
||||
self.encoder = self._build_encoder(
|
||||
config=config.encoder_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "encoder"),
|
||||
@@ -633,6 +632,18 @@ class GraniteSpeechForConditionalGeneration(
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _build_encoder(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> "GraniteSpeechCTCEncoder":
|
||||
return GraniteSpeechCTCEncoder(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self,
|
||||
**kwargs: object,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only IBM Granite Speech Plus model."""
|
||||
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .granite_speech import (
|
||||
GraniteSpeechCTCEncoder,
|
||||
GraniteSpeechDummyInputsBuilder,
|
||||
GraniteSpeechForConditionalGeneration,
|
||||
GraniteSpeechMultiModalProcessingInfo,
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
)
|
||||
|
||||
ISO639_1_SUPPORTED_LANGS = {
|
||||
"en": "English",
|
||||
"fr": "French",
|
||||
"de": "German",
|
||||
"pt": "Portuguese",
|
||||
"es": "Spanish",
|
||||
}
|
||||
|
||||
|
||||
class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
hidden_states = self.input_linear(hidden_states)
|
||||
# cat_hidden_layers selects non-negative layer indices (0 = encoder
|
||||
# input, N = output of layer N) whose hidden states are concatenated
|
||||
# along the feature dim *in addition to* the final hidden states,
|
||||
# which are always appended last.
|
||||
cat_layers = set(self.config.cat_hidden_layers or [])
|
||||
exported_hidden_states = []
|
||||
|
||||
if 0 in cat_layers:
|
||||
exported_hidden_states.append(hidden_states)
|
||||
|
||||
for idx, layer in enumerate(self.layers, start=1):
|
||||
hidden_states = layer(hidden_states, attention_dists=self.attention_dists)
|
||||
|
||||
# Skip the final layer here since its output is always appended
|
||||
# below; capturing it twice would double-append.
|
||||
if idx in cat_layers and idx != self.num_layers:
|
||||
exported_hidden_states.append(hidden_states)
|
||||
|
||||
if idx == self.num_layers // 2:
|
||||
hidden_states_mid = hidden_states.clone()
|
||||
hidden_states_mid, _ = self.out(hidden_states_mid)
|
||||
hidden_states_mid = self.softmax(hidden_states_mid)
|
||||
hidden_states_mid, _ = self.out_mid(hidden_states_mid)
|
||||
hidden_states += hidden_states_mid
|
||||
|
||||
if exported_hidden_states:
|
||||
hidden_states = torch.cat([*exported_hidden_states, hidden_states], dim=-1)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
info=GraniteSpeechMultiModalProcessingInfo,
|
||||
dummy_inputs=GraniteSpeechDummyInputsBuilder,
|
||||
)
|
||||
class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
|
||||
supported_languages = ISO639_1_SUPPORTED_LANGS
|
||||
|
||||
def _build_encoder(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> GraniteSpeechCTCEncoder:
|
||||
return GraniteSpeechPlusCTCEncoder(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
@@ -419,6 +419,10 @@ _MULTIMODAL_MODELS = {
|
||||
"granite_speech",
|
||||
"GraniteSpeechForConditionalGeneration",
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": (
|
||||
"granite_speech_plus",
|
||||
"GraniteSpeechPlusForConditionalGeneration",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": (
|
||||
"granite4_vision",
|
||||
"Granite4VisionForConditionalGeneration",
|
||||
|
||||
Reference in New Issue
Block a user