[Frontend] Add automatic language detection for Whisper transcription (#34342)
Signed-off-by: space_check <roman.vuskov@rwth-aachen.de> Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Nicolò Lucchesi
parent
272b535ab3
commit
98b0205c3c
@@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol):
|
||||
Enables the segment timestamp option for supported models by setting this to `True`.
|
||||
"""
|
||||
|
||||
supports_explicit_language_detection: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models that require an explicit language detection step
|
||||
(e.g. Whisper needs a separate forward pass to predict the language
|
||||
token) should set this to ``True`` and implement
|
||||
:meth:`get_language_detection_prompt` and
|
||||
:meth:`parse_language_detection_output` and
|
||||
:meth:`get_language_token_ids`.
|
||||
"""
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
# language codes in supported_languages
|
||||
@@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol):
|
||||
"""
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def get_language_detection_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
) -> PromptType:
|
||||
"""Return a prompt that triggers language detection.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def parse_language_detection_output(
|
||||
cls,
|
||||
token_ids: list[int],
|
||||
tokenizer: object,
|
||||
) -> str:
|
||||
"""Parse the detected language from model output token IDs.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_language_token_ids(
|
||||
cls,
|
||||
tokenizer: object,
|
||||
) -> list[int] | None:
|
||||
"""Return token IDs that represent valid language tokens.
|
||||
|
||||
Used to constrain language detection to only produce valid language tokens.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(
|
||||
|
||||
Reference in New Issue
Block a user