From fa4bec90567715d1482b01fb6ad6c16bc5c1ed36 Mon Sep 17 00:00:00 2001 From: Andy Lo Date: Wed, 1 Jul 2026 10:33:19 +0100 Subject: [PATCH] [Bugfix] Fix pooled Whisper sliding-window KV sizing (#47071) Signed-off-by: Andy Lo --- .../generation/test_voxtral_realtime.py | 33 ++++++++++++++++++- vllm/model_executor/models/whisper_causal.py | 12 ++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index be677ccb57..149095609e 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -15,11 +15,14 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy from vllm import LLM, EngineArgs, SamplingParams from vllm.assets.audio import AudioAsset from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.utils.math_utils import cdiv from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.kv_cache_interface import SlidingWindowSpec from ....utils import ROCM_ENGINE_KWARGS MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602" +AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn" ENGINE_CONFIG = { "model": MODEL_NAME, "max_model_len": 8192, @@ -60,6 +63,31 @@ def _normalize(texts: list[str]) -> list[str]: return texts +def assert_encoder_kv_cache_spec(engine: LLM) -> None: + vllm_config = engine.llm_engine.vllm_config + audio_config = vllm_config.model_config.hf_config.audio_config + kv_cache_specs_per_rank = engine.llm_engine.model_executor.get_kv_cache_specs() + + assert len(kv_cache_specs_per_rank) == 1 + kv_cache_specs = kv_cache_specs_per_rank[0] + assert AUDIO_LAYER_NAME in kv_cache_specs, kv_cache_specs.keys() + spec = kv_cache_specs[AUDIO_LAYER_NAME] + + assert audio_config.sliding_window == 750 + assert audio_config.block_pool_size == 4 + assert isinstance(spec, SlidingWindowSpec) + assert spec.block_size == 16 + assert spec.num_kv_heads == 128 + assert spec.sliding_window == cdiv(750, 4) == 188 + assert ( + spec.max_admission_blocks_per_request( + max_num_batched_tokens=1, + max_model_len=vllm_config.model_config.max_model_len, + ) + == 13 + ) + + @pytest.fixture def audio_assets() -> list[AudioAsset]: return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")] @@ -71,7 +99,9 @@ def tokenizer() -> MistralTokenizer: @pytest.fixture -def engine(): +def engine(monkeypatch: pytest.MonkeyPatch): + # Disable multiprocessing allows us to access model executor from LLM engine + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") engine_args = EngineArgs(**ENGINE_CONFIG) llm = LLM.from_engine_args(engine_args) try: @@ -95,6 +125,7 @@ async def async_engine(): def test_voxtral_realtime_forward(audio_assets, tokenizer, engine): + assert_encoder_kv_cache_spec(engine) audio_config = tokenizer.instruct_tokenizer.tokenizer.audio def from_file(file_path: str): diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index dfbf69418a..575be61831 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.models.mistral import MistralMLP from vllm.model_executor.models.whisper import WhisperPosEmbedType +from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import ( AttentionBackend, AttentionMetadata, @@ -39,7 +40,7 @@ except ImportError: from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend from vllm.v1.attention.selector import get_attn_backend -from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec from .utils import make_layers @@ -329,6 +330,15 @@ class WhisperCausalAttentionWithBlockPooling(Attention): kv_cache_spec, num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads, ) + if isinstance(kv_cache_spec, SlidingWindowSpec): + # The KV cache manager counts blocks in pooled units, so express the + # window in pooled units too to avoid reserving `block_pool_size`x + # too many blocks. The kernel window is unaffected because it comes + # from the attention impl, not this spec. + kv_cache_spec = replace( + kv_cache_spec, + sliding_window=cdiv(kv_cache_spec.sliding_window, self.block_pool_size), + ) return kv_cache_spec