[Bugfix] Take the sliding window from the layer, not the KV cache group (#51756)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-11 09:10:46 -07:00
committed by GitHub
parent c65aa2ee6c
commit 513f83e7ec
4 changed files with 116 additions and 26 deletions
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""A KV cache group may hold both windowed and global layers — e.g. Gemma-3 with
the hybrid KV cache manager disabled, where the sliding-window layers are
promoted to full-attention *storage* and merged into one group whose spec still
records the window. Backends must take the window from the layers, never from
that group spec, or the global layers get windowed too and silently lose access
to everything older than the window.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from vllm.v1.attention.backends.cpu_attn import (
CPUAttentionBackendImpl,
CPUAttentionMetadataBuilder,
)
def _layers(layer_windows: list[int]):
"""Stand-in attention layers, one per window, as one KV cache group."""
return {
f"layer_{i}": SimpleNamespace(
impl=MagicMock(spec=CPUAttentionBackendImpl, sliding_window=window)
)
for i, window in enumerate(layer_windows)
}
@pytest.mark.parametrize(
"layer_windows,expected",
[
([512, 512], 512), # uniform sliding window -> shared by the group
([-1, -1], -1), # all global -> no window
([512, -1], -1), # mixed -> the group cannot assume either window
],
)
def test_cpu_group_sliding_window(layer_windows, expected):
layers = _layers(layer_windows)
builder = SimpleNamespace(vllm_config=None, layer_names=list(layers))
with patch(
"vllm.v1.attention.backends.cpu_attn.get_layers_from_vllm_config",
return_value=layers,
):
window = CPUAttentionMetadataBuilder._group_sliding_window(builder)
assert window == expected
@@ -83,6 +83,34 @@ def test_sliding_window_retrieval(
)
@pytest.mark.parametrize("model", ["google/gemma-3-1b-it"])
def test_hybrid_kv_cache_manager_output_equivalence(model, vllm_runner):
"""Disabling the hybrid KV cache manager must not change what the model
computes: it promotes the sliding-window layers to full-attention *storage*
only, so the global layers must keep attending globally.
Guards against the group's KV cache spec, which then covers windowed and
global layers alike, imposing its window on the global layers too.
"""
prompts, _, _ = prep_prompts(2, ln_range=model_config[model].ln_range)
sampling_params = SamplingParams(temperature=0.0, max_tokens=32)
enforce_eager = current_platform.is_rocm()
outputs = []
for disable_hybrid in (False, True):
with vllm_runner(
model,
max_model_len=None,
enable_chunked_prefill=None,
disable_hybrid_kv_cache_manager=disable_hybrid,
enforce_eager=enforce_eager,
) as runner:
responses = runner.get_llm().generate(prompts, sampling_params)
outputs.append([response.outputs[0].text for response in responses])
assert outputs[0] == outputs[1]
def check_length(prompts: list[str], llm: LLM, sliding_window: int):
"""
Check if the prompt length is valid, i.e., longer than the sliding window