[Frontend] Add automatic language detection for Whisper transcription (#34342)
Signed-off-by: space_check <roman.vuskov@rwth-aachen.de> Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Nicolò Lucchesi
parent
272b535ab3
commit
98b0205c3c
@@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol):
|
||||
Enables the segment timestamp option for supported models by setting this to `True`.
|
||||
"""
|
||||
|
||||
supports_explicit_language_detection: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models that require an explicit language detection step
|
||||
(e.g. Whisper needs a separate forward pass to predict the language
|
||||
token) should set this to ``True`` and implement
|
||||
:meth:`get_language_detection_prompt` and
|
||||
:meth:`parse_language_detection_output` and
|
||||
:meth:`get_language_token_ids`.
|
||||
"""
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
# language codes in supported_languages
|
||||
@@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol):
|
||||
"""
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def get_language_detection_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
) -> PromptType:
|
||||
"""Return a prompt that triggers language detection.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def parse_language_detection_output(
|
||||
cls,
|
||||
token_ids: list[int],
|
||||
tokenizer: object,
|
||||
) -> str:
|
||||
"""Parse the detected language from model output token IDs.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_language_token_ids(
|
||||
cls,
|
||||
tokenizer: object,
|
||||
) -> list[int] | None:
|
||||
"""Return token IDs that represent valid language tokens.
|
||||
|
||||
Used to constrain language detection to only produce valid language tokens.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(
|
||||
|
||||
@@ -64,7 +64,11 @@ from vllm.v1.attention.backend import (
|
||||
AttentionType,
|
||||
)
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModal,
|
||||
SupportsTranscription,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -784,7 +788,9 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
dummy_inputs=WhisperDummyInputsBuilder,
|
||||
)
|
||||
class WhisperForConditionalGeneration(
|
||||
nn.Module, SupportsTranscription, SupportsMultiModal
|
||||
nn.Module,
|
||||
SupportsTranscription,
|
||||
SupportsMultiModal,
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"self_attn.qkv_proj": [
|
||||
@@ -802,20 +808,18 @@ class WhisperForConditionalGeneration(
|
||||
# Whisper only supports audio-conditioned generation.
|
||||
supports_transcription_only = True
|
||||
supports_segment_timestamp = True
|
||||
supports_explicit_language_detection = True
|
||||
supported_languages = ISO639_1_SUPPORTED_LANGS
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
if language is None:
|
||||
# TODO language should be optional and can be guessed.
|
||||
# For now we default to en. See
|
||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
|
||||
logger.warning(
|
||||
"Defaulting to language='en'. If you wish to transcribe "
|
||||
"audio in a different language, pass the `language` field "
|
||||
logger.debug(
|
||||
"No language specified. Language will be auto-detected "
|
||||
"from audio. To skip detection, pass the `language` field "
|
||||
"in the TranscriptionRequest."
|
||||
)
|
||||
language = "en"
|
||||
return None
|
||||
return super().validate_language(language)
|
||||
|
||||
@classmethod
|
||||
@@ -846,6 +850,63 @@ class WhisperForConditionalGeneration(
|
||||
decoder_prompt=TextPrompt(prompt=decoder_text),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_language_token_ids(
|
||||
cls,
|
||||
tokenizer: object,
|
||||
) -> list[int]:
|
||||
"""Return token IDs for all supported language tokens.
|
||||
|
||||
Used with ``SamplingParams.allowed_token_ids`` to constrain
|
||||
language detection to only produce valid language tokens.
|
||||
"""
|
||||
token_ids = [
|
||||
tokenizer.convert_tokens_to_ids(f"<|{lang_code}|>")
|
||||
for lang_code in cls.supported_languages
|
||||
]
|
||||
return token_ids
|
||||
|
||||
@classmethod
|
||||
def get_language_detection_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
) -> PromptType:
|
||||
"""Return a prompt that elicits a single language token from Whisper.
|
||||
|
||||
Feed only ``<|startoftranscript|>`` as the decoder input so the model
|
||||
predicts the most likely language token (e.g. ``<|de|>``).
|
||||
"""
|
||||
return ExplicitEncoderDecoderPrompt(
|
||||
encoder_prompt=TextPrompt(
|
||||
prompt="",
|
||||
multi_modal_data={"audio": (audio, stt_config.sample_rate)},
|
||||
),
|
||||
decoder_prompt=TextPrompt(prompt="<|startoftranscript|>"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def parse_language_detection_output(
|
||||
cls,
|
||||
token_ids: list[int],
|
||||
tokenizer: object,
|
||||
) -> str | None:
|
||||
"""Parse the language token predicted by Whisper.
|
||||
|
||||
Decodes the first token ID and extracts the language code from the
|
||||
``<|xx|>`` format. Expects a valid language token from constrained generation.
|
||||
"""
|
||||
|
||||
decoded = tokenizer.decode(
|
||||
[token_ids[0]],
|
||||
skip_special_tokens=False,
|
||||
)
|
||||
# Whisper language tokens have the form <|xx|>
|
||||
assert decoded.startswith("<|") and decoded.endswith("|>")
|
||||
lang_code = decoded[2:-2]
|
||||
assert lang_code in cls.supported_languages
|
||||
return lang_code
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("audio"):
|
||||
|
||||
Reference in New Issue
Block a user