[Bugfix] Take the sliding window from the layer, not the KV cache group (#51756)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""A KV cache group may hold both windowed and global layers — e.g. Gemma-3 with
|
||||
the hybrid KV cache manager disabled, where the sliding-window layers are
|
||||
promoted to full-attention *storage* and merged into one group whose spec still
|
||||
records the window. Backends must take the window from the layers, never from
|
||||
that group spec, or the global layers get windowed too and silently lose access
|
||||
to everything older than the window.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.attention.backends.cpu_attn import (
|
||||
CPUAttentionBackendImpl,
|
||||
CPUAttentionMetadataBuilder,
|
||||
)
|
||||
|
||||
|
||||
def _layers(layer_windows: list[int]):
|
||||
"""Stand-in attention layers, one per window, as one KV cache group."""
|
||||
return {
|
||||
f"layer_{i}": SimpleNamespace(
|
||||
impl=MagicMock(spec=CPUAttentionBackendImpl, sliding_window=window)
|
||||
)
|
||||
for i, window in enumerate(layer_windows)
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_windows,expected",
|
||||
[
|
||||
([512, 512], 512), # uniform sliding window -> shared by the group
|
||||
([-1, -1], -1), # all global -> no window
|
||||
([512, -1], -1), # mixed -> the group cannot assume either window
|
||||
],
|
||||
)
|
||||
def test_cpu_group_sliding_window(layer_windows, expected):
|
||||
layers = _layers(layer_windows)
|
||||
builder = SimpleNamespace(vllm_config=None, layer_names=list(layers))
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.cpu_attn.get_layers_from_vllm_config",
|
||||
return_value=layers,
|
||||
):
|
||||
window = CPUAttentionMetadataBuilder._group_sliding_window(builder)
|
||||
assert window == expected
|
||||
@@ -83,6 +83,34 @@ def test_sliding_window_retrieval(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["google/gemma-3-1b-it"])
|
||||
def test_hybrid_kv_cache_manager_output_equivalence(model, vllm_runner):
|
||||
"""Disabling the hybrid KV cache manager must not change what the model
|
||||
computes: it promotes the sliding-window layers to full-attention *storage*
|
||||
only, so the global layers must keep attending globally.
|
||||
|
||||
Guards against the group's KV cache spec, which then covers windowed and
|
||||
global layers alike, imposing its window on the global layers too.
|
||||
"""
|
||||
prompts, _, _ = prep_prompts(2, ln_range=model_config[model].ln_range)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=32)
|
||||
enforce_eager = current_platform.is_rocm()
|
||||
|
||||
outputs = []
|
||||
for disable_hybrid in (False, True):
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=None,
|
||||
enable_chunked_prefill=None,
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid,
|
||||
enforce_eager=enforce_eager,
|
||||
) as runner:
|
||||
responses = runner.get_llm().generate(prompts, sampling_params)
|
||||
outputs.append([response.outputs[0].text for response in responses])
|
||||
|
||||
assert outputs[0] == outputs[1]
|
||||
|
||||
|
||||
def check_length(prompts: list[str], llm: LLM, sliding_window: int):
|
||||
"""
|
||||
Check if the prompt length is valid, i.e., longer than the sliding window
|
||||
|
||||
Reference in New Issue
Block a user