[Frontend] Add automatic language detection for Whisper transcription (#34342)

Signed-off-by: space_check <roman.vuskov@rwth-aachen.de>
Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Roman
2026-02-21 04:49:41 -08:00
committed by GitHub
co-authored by Nicolò Lucchesi Nicolò Lucchesi
parent 272b535ab3
commit 98b0205c3c
5 changed files with 249 additions and 10 deletions
+50
View File
@@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol):
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
(e.g. Whisper needs a separate forward pass to predict the language
token) should set this to ``True`` and implement
:meth:`get_language_detection_prompt` and
:meth:`parse_language_detection_output` and
:meth:`get_language_token_ids`.
"""
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# language codes in supported_languages
@@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def get_language_detection_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
) -> PromptType:
"""Return a prompt that triggers language detection.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
) -> str:
"""Parse the detected language from model output token IDs.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
Used to constrain language detection to only produce valid language tokens.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@overload
def supports_transcription(