From 443e68cfa616312d88aecd613a8013d5b5169956 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 2 Jul 2026 18:19:11 +0100 Subject: [PATCH] [Bugfix] Fix pooled Whisper encoder sliding-window kernel size (#47437) Signed-off-by: Nick Hill --- .../generation/test_voxtral_realtime.py | 4 ++- vllm/model_executor/models/whisper_causal.py | 36 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 149095609e..9fb601df02 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -78,7 +78,9 @@ def assert_encoder_kv_cache_spec(engine: LLM) -> None: assert isinstance(spec, SlidingWindowSpec) assert spec.block_size == 16 assert spec.num_kv_heads == 128 - assert spec.sliding_window == cdiv(750, 4) == 188 + # cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra + # +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec). + assert spec.sliding_window == cdiv(750, 4) + 1 == 189 assert ( spec.max_admission_blocks_per_request( max_num_batched_tokens=1, diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index 575be61831..91b378e0dc 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -112,7 +112,9 @@ class WhisperCausalConv1d(nn.Conv1d): @functools.lru_cache def create_whisper_attention_backend_with_block_pooling( - underlying_attn_backend: AttentionBackend, block_pool_size: int + underlying_attn_backend: AttentionBackend, + block_pool_size: int, + sliding_window: int | None = None, ) -> type[AttentionBackend]: prefix = "WhisperCausalAttentionWithBlockPooling_" underlying_builder = underlying_attn_backend.get_builder_cls() @@ -127,11 +129,22 @@ def create_whisper_attention_backend_with_block_pooling( device: torch.device, ): assert kv_cache_spec.num_kv_heads % block_pool_size == 0 - kv_cache_spec = replace( - kv_cache_spec, + # Scale pooled-unit quantities back to unpooled (encoder-token) + # units for the underlying attention metadata: the KV cache spec + # counts blocks in pooled units, but the kernel runs on the + # `block_pool_size`x-expanded sequence built below. + spec_overrides = dict( block_size=kv_cache_spec.block_size * block_pool_size, num_kv_heads=kv_cache_spec.num_kv_heads // block_pool_size, ) + if isinstance(kv_cache_spec, SlidingWindowSpec): + # The manager keeps `sliding_window` in pooled units; the kernel + # runs on the expanded sequence, so give it the model's window in + # unpooled units (`get_kv_cache_spec` sizes the pooled window so + # this window always stays resident). + assert sliding_window is not None + spec_overrides["sliding_window"] = sliding_window + kv_cache_spec = replace(kv_cache_spec, **spec_overrides) super().__init__(kv_cache_spec, layer_names, vllm_config, device) # Override model_config-derived values with the actual # encoder values from kv_cache_spec @@ -303,7 +316,7 @@ class WhisperCausalAttentionWithBlockPooling(Attention): attn_type=attn_type, ) attn_backend = create_whisper_attention_backend_with_block_pooling( - underlying_attn_backend, block_pool_size + underlying_attn_backend, block_pool_size, per_layer_sliding_window ) super().__init__( @@ -331,14 +344,13 @@ class WhisperCausalAttentionWithBlockPooling(Attention): num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads, ) if isinstance(kv_cache_spec, SlidingWindowSpec): - # The KV cache manager counts blocks in pooled units, so express the - # window in pooled units too to avoid reserving `block_pool_size`x - # too many blocks. The kernel window is unaffected because it comes - # from the attention impl, not this spec. - kv_cache_spec = replace( - kv_cache_spec, - sliding_window=cdiv(kv_cache_spec.sliding_window, self.block_pool_size), - ) + # The manager counts blocks in pooled units, so express the window in + # pooled units to avoid reserving `block_pool_size`x too many blocks. + # The `+1` is an eviction margin: block-aligned eviction only keeps + # `pooled - 1` pooled tokens, so this guarantees the kernel's full + # (unpooled) window still stays resident. + pooled = cdiv(kv_cache_spec.sliding_window, self.block_pool_size) + 1 + kv_cache_spec = replace(kv_cache_spec, sliding_window=pooled) return kv_cache_spec