[Core] Support multiple tasks per model (#20771)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
c1acd6d7d4
commit
020f58abcd
@@ -694,6 +694,12 @@ class SupportsTranscription(Protocol):
|
||||
|
||||
supports_transcription: ClassVar[Literal[True]] = True
|
||||
|
||||
supports_transcription_only: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models can opt out of text generation by setting this to
|
||||
`True`.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(cls, audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig, language: str,
|
||||
|
||||
@@ -284,6 +284,7 @@ class _ModelInfo:
|
||||
is_hybrid: bool
|
||||
has_noops: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_v0_only: bool
|
||||
|
||||
@staticmethod
|
||||
@@ -299,6 +300,8 @@ class _ModelInfo:
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
supports_transcription=supports_transcription(model),
|
||||
supports_transcription_only=(supports_transcription(model) and
|
||||
model.supports_transcription_only),
|
||||
supports_v0_only=supports_v0_only(model),
|
||||
has_noops=has_noops(model),
|
||||
)
|
||||
@@ -573,6 +576,13 @@ class _ModelRegistry:
|
||||
model_cls, _ = self.inspect_model_cls(architectures)
|
||||
return model_cls.supports_transcription
|
||||
|
||||
def is_transcription_only_model(
|
||||
self,
|
||||
architectures: Union[str, list[str]],
|
||||
) -> bool:
|
||||
model_cls, _ = self.inspect_model_cls(architectures)
|
||||
return model_cls.supports_transcription_only
|
||||
|
||||
def is_v1_compatible(
|
||||
self,
|
||||
architectures: Union[str, list[str]],
|
||||
|
||||
@@ -772,6 +772,9 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
|
||||
".fc2.": ".mlp.fc2."
|
||||
})
|
||||
|
||||
# Whisper only supports audio-conditioned generation.
|
||||
supports_transcription_only = True
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str) -> bool:
|
||||
if language in ISO639_1_SUPPORTED_LANGS:
|
||||
|
||||
Reference in New Issue
Block a user