[Bugfix] Fix pooled Whisper encoder sliding-window kernel size (#47437)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -78,7 +78,9 @@ def assert_encoder_kv_cache_spec(engine: LLM) -> None:
|
||||
assert isinstance(spec, SlidingWindowSpec)
|
||||
assert spec.block_size == 16
|
||||
assert spec.num_kv_heads == 128
|
||||
assert spec.sliding_window == cdiv(750, 4) == 188
|
||||
# cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra
|
||||
# +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec).
|
||||
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
|
||||
assert (
|
||||
spec.max_admission_blocks_per_request(
|
||||
max_num_batched_tokens=1,
|
||||
|
||||
@@ -112,7 +112,9 @@ class WhisperCausalConv1d(nn.Conv1d):
|
||||
|
||||
@functools.lru_cache
|
||||
def create_whisper_attention_backend_with_block_pooling(
|
||||
underlying_attn_backend: AttentionBackend, block_pool_size: int
|
||||
underlying_attn_backend: AttentionBackend,
|
||||
block_pool_size: int,
|
||||
sliding_window: int | None = None,
|
||||
) -> type[AttentionBackend]:
|
||||
prefix = "WhisperCausalAttentionWithBlockPooling_"
|
||||
underlying_builder = underlying_attn_backend.get_builder_cls()
|
||||
@@ -127,11 +129,22 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
device: torch.device,
|
||||
):
|
||||
assert kv_cache_spec.num_kv_heads % block_pool_size == 0
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
# Scale pooled-unit quantities back to unpooled (encoder-token)
|
||||
# units for the underlying attention metadata: the KV cache spec
|
||||
# counts blocks in pooled units, but the kernel runs on the
|
||||
# `block_pool_size`x-expanded sequence built below.
|
||||
spec_overrides = dict(
|
||||
block_size=kv_cache_spec.block_size * block_pool_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads // block_pool_size,
|
||||
)
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
# The manager keeps `sliding_window` in pooled units; the kernel
|
||||
# runs on the expanded sequence, so give it the model's window in
|
||||
# unpooled units (`get_kv_cache_spec` sizes the pooled window so
|
||||
# this window always stays resident).
|
||||
assert sliding_window is not None
|
||||
spec_overrides["sliding_window"] = sliding_window
|
||||
kv_cache_spec = replace(kv_cache_spec, **spec_overrides)
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
# Override model_config-derived values with the actual
|
||||
# encoder values from kv_cache_spec
|
||||
@@ -303,7 +316,7 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
|
||||
attn_type=attn_type,
|
||||
)
|
||||
attn_backend = create_whisper_attention_backend_with_block_pooling(
|
||||
underlying_attn_backend, block_pool_size
|
||||
underlying_attn_backend, block_pool_size, per_layer_sliding_window
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
@@ -331,14 +344,13 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
|
||||
num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads,
|
||||
)
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
# The KV cache manager counts blocks in pooled units, so express the
|
||||
# window in pooled units too to avoid reserving `block_pool_size`x
|
||||
# too many blocks. The kernel window is unaffected because it comes
|
||||
# from the attention impl, not this spec.
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
sliding_window=cdiv(kv_cache_spec.sliding_window, self.block_pool_size),
|
||||
)
|
||||
# The manager counts blocks in pooled units, so express the window in
|
||||
# pooled units to avoid reserving `block_pool_size`x too many blocks.
|
||||
# The `+1` is an eviction margin: block-aligned eviction only keeps
|
||||
# `pooled - 1` pooled tokens, so this guarantees the kernel's full
|
||||
# (unpooled) window still stays resident.
|
||||
pooled = cdiv(kv_cache_spec.sliding_window, self.block_pool_size) + 1
|
||||
kv_cache_spec = replace(kv_cache_spec, sliding_window=pooled)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user