[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from mistral_common.protocol.transcription.request import (
|
||||
@@ -75,7 +77,8 @@ def assert_encoder_kv_cache_spec(engine: LLM) -> None:
|
||||
assert audio_config.block_pool_size == 4
|
||||
assert isinstance(spec, SlidingWindowSpec)
|
||||
assert spec.block_size == 16
|
||||
assert spec.num_kv_heads == 128
|
||||
assert spec.num_kv_heads == 32
|
||||
assert spec.tokens_per_state == Fraction(1, 4)
|
||||
# cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra
|
||||
# +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec).
|
||||
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_configs,
|
||||
@@ -82,6 +83,12 @@ def can_initialize(
|
||||
# Avoid calling model.forward()
|
||||
def _initialize_kv_caches_v1(self, vllm_config):
|
||||
kv_cache_specs = self.model_executor.get_kv_cache_specs()
|
||||
layout = resolve_kv_cache_layout(
|
||||
vllm_config,
|
||||
self.model_executor.get_supported_kv_cache_layouts(),
|
||||
[spec for worker_specs in kv_cache_specs for spec in worker_specs.values()],
|
||||
)
|
||||
self.model_executor.set_kv_cache_layout(layout.name)
|
||||
kv_cache_configs = get_kv_cache_configs(
|
||||
vllm_config,
|
||||
kv_cache_specs,
|
||||
|
||||
Reference in New Issue
Block a user