[Frontend] Add automatic language detection for Whisper transcription (#34342)

Signed-off-by: space_check <roman.vuskov@rwth-aachen.de>
Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Roman
2026-02-21 04:49:41 -08:00
committed by GitHub
co-authored by Nicolò Lucchesi Nicolò Lucchesi
parent 272b535ab3
commit 98b0205c3c
5 changed files with 249 additions and 10 deletions
+50
View File
@@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol):
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
(e.g. Whisper needs a separate forward pass to predict the language
token) should set this to ``True`` and implement
:meth:`get_language_detection_prompt` and
:meth:`parse_language_detection_output` and
:meth:`get_language_token_ids`.
"""
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# language codes in supported_languages
@@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol):
"""
return text
@classmethod
def get_language_detection_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
) -> PromptType:
"""Return a prompt that triggers language detection.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
) -> str:
"""Parse the detected language from model output token IDs.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
Used to constrain language detection to only produce valid language tokens.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@overload
def supports_transcription(
+70 -9
View File
@@ -64,7 +64,11 @@ from vllm.v1.attention.backend import (
AttentionType,
)
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsTranscription,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -784,7 +788,9 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
dummy_inputs=WhisperDummyInputsBuilder,
)
class WhisperForConditionalGeneration(
nn.Module, SupportsTranscription, SupportsMultiModal
nn.Module,
SupportsTranscription,
SupportsMultiModal,
):
packed_modules_mapping = {
"self_attn.qkv_proj": [
@@ -802,20 +808,18 @@ class WhisperForConditionalGeneration(
# Whisper only supports audio-conditioned generation.
supports_transcription_only = True
supports_segment_timestamp = True
supports_explicit_language_detection = True
supported_languages = ISO639_1_SUPPORTED_LANGS
@classmethod
def validate_language(cls, language: str | None) -> str | None:
if language is None:
# TODO language should be optional and can be guessed.
# For now we default to en. See
# https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
logger.warning(
"Defaulting to language='en'. If you wish to transcribe "
"audio in a different language, pass the `language` field "
logger.debug(
"No language specified. Language will be auto-detected "
"from audio. To skip detection, pass the `language` field "
"in the TranscriptionRequest."
)
language = "en"
return None
return super().validate_language(language)
@classmethod
@@ -846,6 +850,63 @@ class WhisperForConditionalGeneration(
decoder_prompt=TextPrompt(prompt=decoder_text),
)
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
) -> list[int]:
"""Return token IDs for all supported language tokens.
Used with ``SamplingParams.allowed_token_ids`` to constrain
language detection to only produce valid language tokens.
"""
token_ids = [
tokenizer.convert_tokens_to_ids(f"<|{lang_code}|>")
for lang_code in cls.supported_languages
]
return token_ids
@classmethod
def get_language_detection_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
) -> PromptType:
"""Return a prompt that elicits a single language token from Whisper.
Feed only ``<|startoftranscript|>`` as the decoder input so the model
predicts the most likely language token (e.g. ``<|de|>``).
"""
return ExplicitEncoderDecoderPrompt(
encoder_prompt=TextPrompt(
prompt="",
multi_modal_data={"audio": (audio, stt_config.sample_rate)},
),
decoder_prompt=TextPrompt(prompt="<|startoftranscript|>"),
)
@classmethod
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
) -> str | None:
"""Parse the language token predicted by Whisper.
Decodes the first token ID and extracts the language code from the
``<|xx|>`` format. Expects a valid language token from constrained generation.
"""
decoded = tokenizer.decode(
[token_ids[0]],
skip_special_tokens=False,
)
# Whisper language tokens have the form <|xx|>
assert decoded.startswith("<|") and decoded.endswith("|>")
lang_code = decoded[2:-2]
assert lang_code in cls.supported_languages
return lang_code
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("audio"):