[Core] Support multiple tasks per model (#20771)

Signed-off-by: NickLucche <nlucches@redhat.com>
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Nicolò Lucchesi
2025-07-12 19:40:11 -07:00
committed by GitHub
co-authored by DarkLight1337
parent c1acd6d7d4
commit 020f58abcd
8 changed files with 279 additions and 148 deletions
+6
View File
@@ -694,6 +694,12 @@ class SupportsTranscription(Protocol):
supports_transcription: ClassVar[Literal[True]] = True
supports_transcription_only: ClassVar[bool] = False
"""
Transcription models can opt out of text generation by setting this to
`True`.
"""
@classmethod
def get_generation_prompt(cls, audio: np.ndarray,
stt_config: SpeechToTextConfig, language: str,
+10
View File
@@ -284,6 +284,7 @@ class _ModelInfo:
is_hybrid: bool
has_noops: bool
supports_transcription: bool
supports_transcription_only: bool
supports_v0_only: bool
@staticmethod
@@ -299,6 +300,8 @@ class _ModelInfo:
is_attention_free=is_attention_free(model),
is_hybrid=is_hybrid(model),
supports_transcription=supports_transcription(model),
supports_transcription_only=(supports_transcription(model) and
model.supports_transcription_only),
supports_v0_only=supports_v0_only(model),
has_noops=has_noops(model),
)
@@ -573,6 +576,13 @@ class _ModelRegistry:
model_cls, _ = self.inspect_model_cls(architectures)
return model_cls.supports_transcription
def is_transcription_only_model(
self,
architectures: Union[str, list[str]],
) -> bool:
model_cls, _ = self.inspect_model_cls(architectures)
return model_cls.supports_transcription_only
def is_v1_compatible(
self,
architectures: Union[str, list[str]],
+3
View File
@@ -772,6 +772,9 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
".fc2.": ".mlp.fc2."
})
# Whisper only supports audio-conditioned generation.
supports_transcription_only = True
@classmethod
def validate_language(cls, language: str) -> bool:
if language in ISO639_1_SUPPORTED_LANGS: