[Bugfix][Model] Fix CohereASR streaming audio-token estimate (unit + subsampling) (#53829)
Signed-off-by: hungh <hungh@nvidia.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
41848caa63
commit
3b45d053b4
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import SpeechToTextConfig
|
||||
from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("audio_duration_s", "expected_tokens"),
|
||||
# window_stride=0.01s @ 16kHz -> 160-sample hop; subsampling_factor=8.
|
||||
# frames = floor(duration * 16000 / 160); tokens = ceil(frames / 8).
|
||||
[(1.0, 13), (10.0, 125), (30.0, 375)],
|
||||
)
|
||||
def test_get_num_audio_tokens_streaming_estimate(audio_duration_s, expected_tokens):
|
||||
"""The duration-based estimate must convert ``window_stride`` (seconds) to a
|
||||
sample hop and divide by the encoder subsampling factor. Values are pinned to
|
||||
concrete numbers so a rounding regression is actually caught."""
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
preprocessor={"window_stride": 0.01, "sample_rate": 16000},
|
||||
encoder={"subsampling_factor": 8},
|
||||
)
|
||||
)
|
||||
stt_config = SpeechToTextConfig(sample_rate=16000)
|
||||
|
||||
got = CohereAsrForConditionalGeneration.get_num_audio_tokens(
|
||||
audio_duration_s, stt_config, model_config
|
||||
)
|
||||
|
||||
assert got == expected_tokens
|
||||
@@ -2177,9 +2177,15 @@ class CohereAsrForConditionalGeneration(
|
||||
stt_config: SpeechToTextConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> int | None:
|
||||
hop_length = model_config.hf_config.preprocessor.get("window_stride")
|
||||
assert hop_length is not None
|
||||
return math.ceil(audio_duration_s * stt_config.sample_rate / hop_length)
|
||||
hf_config = model_config.hf_config
|
||||
preprocessor = hf_config.preprocessor
|
||||
sample_rate = preprocessor["sample_rate"]
|
||||
window_stride = preprocessor["window_stride"]
|
||||
hop_length = int(window_stride * sample_rate)
|
||||
# Floor-divide to match get_seq_len.
|
||||
num_frames = int(audio_duration_s * sample_rate) // hop_length
|
||||
subsampling_factor = hf_config.encoder["subsampling_factor"]
|
||||
return math.ceil(num_frames / subsampling_factor)
|
||||
|
||||
def get_num_encoder_cross_attn_tokens(self, num_encoder_input_tokens: int) -> int:
|
||||
return self.model.encoder.get_num_encoder_cross_attn_tokens(
|
||||
|
||||
Reference in New Issue
Block a user