[Bugfix] Fix pooled Whisper encoder sliding-window kernel size (#47437)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-02 10:19:11 -07:00
committed by GitHub
parent 320ee285c9
commit 443e68cfa6
2 changed files with 27 additions and 13 deletions
@@ -78,7 +78,9 @@ def assert_encoder_kv_cache_spec(engine: LLM) -> None:
assert isinstance(spec, SlidingWindowSpec)
assert spec.block_size == 16
assert spec.num_kv_heads == 128
assert spec.sliding_window == cdiv(750, 4) == 188
# cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra
# +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec).
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
assert (
spec.max_admission_blocks_per_request(
max_num_batched_tokens=1,
+24 -12
View File
@@ -112,7 +112,9 @@ class WhisperCausalConv1d(nn.Conv1d):
@functools.lru_cache
def create_whisper_attention_backend_with_block_pooling(
underlying_attn_backend: AttentionBackend, block_pool_size: int
underlying_attn_backend: AttentionBackend,
block_pool_size: int,
sliding_window: int | None = None,
) -> type[AttentionBackend]:
prefix = "WhisperCausalAttentionWithBlockPooling_"
underlying_builder = underlying_attn_backend.get_builder_cls()
@@ -127,11 +129,22 @@ def create_whisper_attention_backend_with_block_pooling(
device: torch.device,
):
assert kv_cache_spec.num_kv_heads % block_pool_size == 0
kv_cache_spec = replace(
kv_cache_spec,
# Scale pooled-unit quantities back to unpooled (encoder-token)
# units for the underlying attention metadata: the KV cache spec
# counts blocks in pooled units, but the kernel runs on the
# `block_pool_size`x-expanded sequence built below.
spec_overrides = dict(
block_size=kv_cache_spec.block_size * block_pool_size,
num_kv_heads=kv_cache_spec.num_kv_heads // block_pool_size,
)
if isinstance(kv_cache_spec, SlidingWindowSpec):
# The manager keeps `sliding_window` in pooled units; the kernel
# runs on the expanded sequence, so give it the model's window in
# unpooled units (`get_kv_cache_spec` sizes the pooled window so
# this window always stays resident).
assert sliding_window is not None
spec_overrides["sliding_window"] = sliding_window
kv_cache_spec = replace(kv_cache_spec, **spec_overrides)
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
# Override model_config-derived values with the actual
# encoder values from kv_cache_spec
@@ -303,7 +316,7 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
attn_type=attn_type,
)
attn_backend = create_whisper_attention_backend_with_block_pooling(
underlying_attn_backend, block_pool_size
underlying_attn_backend, block_pool_size, per_layer_sliding_window
)
super().__init__(
@@ -331,14 +344,13 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads,
)
if isinstance(kv_cache_spec, SlidingWindowSpec):
# The KV cache manager counts blocks in pooled units, so express the
# window in pooled units too to avoid reserving `block_pool_size`x
# too many blocks. The kernel window is unaffected because it comes
# from the attention impl, not this spec.
kv_cache_spec = replace(
kv_cache_spec,
sliding_window=cdiv(kv_cache_spec.sliding_window, self.block_pool_size),
)
# The manager counts blocks in pooled units, so express the window in
# pooled units to avoid reserving `block_pool_size`x too many blocks.
# The `+1` is an eviction margin: block-aligned eviction only keeps
# `pooled - 1` pooled tokens, so this guarantees the kernel's full
# (unpooled) window still stays resident.
pooled = cdiv(kv_cache_spec.sliding_window, self.block_pool_size) + 1
kv_cache_spec = replace(kv_cache_spec, sliding_window=pooled)
return kv_cache_spec