From c8bdcc011623de64dbab64813a184ab083730a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ganczarenko?= Date: Wed, 1 Jul 2026 18:42:27 +0300 Subject: [PATCH] [Bench][BugFix] Fix empty decoder prompt for Cohere ASR in throughput benchmark (#47135) Signed-off-by: Michal Ganczarenko --- tests/benchmarks/test_audio_dataset.py | 71 ++++++++++++++++++++++++++ vllm/benchmarks/datasets/datasets.py | 15 +++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/test_audio_dataset.py b/tests/benchmarks/test_audio_dataset.py index 5957011c48..c1b2af7eae 100644 --- a/tests/benchmarks/test_audio_dataset.py +++ b/tests/benchmarks/test_audio_dataset.py @@ -34,6 +34,16 @@ class _Tokenizer: return _TokenizedPrompt(prompt) +class CohereAsrTokenizer(_Tokenizer): + def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None: + super().__init__(name_or_path) + + +class _CohereNameOnlyTokenizer(_Tokenizer): + def __init__(self) -> None: + super().__init__("cohere/some-local-checkpoint") + + def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None: num_samples = int(duration_s * sample_rate) sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate) @@ -198,3 +208,64 @@ def test_async_request_openai_audio_handles_decoded_audio_arrays( assert session.uploaded_bytes is not None assert output.success is True assert output.generated_text == "hello" + + +_COHERE_ASR_PROMPT = ( + "<|startofcontext|><|startoftranscript|>" + "<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>" + "<|notimestamp|><|nodiarize|>" +) + + +def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset: + audio_path = tmp_path / "sample.wav" + _write_wav(audio_path, duration_s=0.1) + dataset = object.__new__(datasets_module.ASRDataset) + dataset.data = [ + { + "audio": {"path": str(audio_path), "bytes": None}, + "text": "hello world", + } + ] + return dataset + + +def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None: + dataset = _make_asr_dataset(tmp_path) + samples = dataset.sample( + tokenizer=CohereAsrTokenizer(), + num_requests=1, + output_len=32, + asr_min_audio_len_sec=0.0, + asr_max_audio_len_sec=1.0, + ) + assert len(samples) == 1 + assert samples[0].prompt == _COHERE_ASR_PROMPT + + +def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt( + tmp_path: Path, +) -> None: + dataset = _make_asr_dataset(tmp_path) + samples = dataset.sample( + tokenizer=_CohereNameOnlyTokenizer(), + num_requests=1, + output_len=32, + asr_min_audio_len_sec=0.0, + asr_max_audio_len_sec=1.0, + ) + assert len(samples) == 1 + assert samples[0].prompt == _COHERE_ASR_PROMPT + + +def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None: + dataset = _make_asr_dataset(tmp_path) + samples = dataset.sample( + tokenizer=_Tokenizer(name_or_path="some-other/asr-model"), + num_requests=1, + output_len=32, + asr_min_audio_len_sec=0.0, + asr_max_audio_len_sec=1.0, + ) + assert len(samples) == 1 + assert samples[0].prompt == "" diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 08c1613d7e..bbfc805702 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -4156,8 +4156,21 @@ class ASRDataset(HuggingFaceDataset): **kwargs, ) -> list[SampleRequest]: output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN - if "openai" in getattr(tokenizer, "name_or_path", ""): + name_or_path = getattr(tokenizer, "name_or_path", "") + tok_class = type(tokenizer).__name__ + if "openai" in name_or_path: prompt = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>" + elif tok_class == "CohereAsrTokenizer" or "cohere" in name_or_path.lower(): + # CohereAsrTokenizer does not inject a decoder start token, so the + # decoder prompt must supply the full control-token sequence. + # Token order: context boundary, transcript start, emotion (default + # undefined), language (en), transcription directive (en), punctuation + # enabled, no ITN, no timestamp, no diarization. + prompt = ( + "<|startofcontext|><|startoftranscript|>" + "<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>" + "<|notimestamp|><|nodiarize|>" + ) else: prompt = "" prompt_len = len(tokenizer(prompt).input_ids)