[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+40
View File
@@ -3,6 +3,7 @@
from collections.abc import Callable
from dataclasses import field
from functools import cache
from typing import Any, ClassVar, Literal
from pydantic import Field, field_validator, model_validator
@@ -13,9 +14,28 @@ from vllm.utils.torch_utils import (
is_quantized_kv_cache,
kv_cache_uses_per_token_head_scales,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
logger = init_logger(__name__)
_LAYOUT_COMPAT_ALIASES = {
"NHD": "LBNHC",
"HND": "LBHNC",
}
@cache
def _layout_from_name(layout_name: str) -> KVCacheLayout:
layout_name = _LAYOUT_COMPAT_ALIASES.get(layout_name, layout_name)
try:
return KVCacheLayout[layout_name]
except KeyError:
raise ValueError(
f"Unknown KV cache layout {layout_name!r}. "
f"Valid layouts: {[m.name for m in KVCacheLayout]}"
) from None
CacheDType = Literal[
"auto",
"float16",
@@ -65,6 +85,17 @@ class CacheConfig:
"""Whether block_size was explicitly provided. Derived automatically."""
user_specified_mamba_block_size: bool = field(default=False, init=False)
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
kv_cache_layout: str | None = field(default=None, init=False)
"""Resolved physical KV cache layout name (a ``KVCacheLayout`` member).
``None`` means the layout has not been resolved yet. The engine core
resolves it once (``resolve_kv_cache_layout``) before memory profiling, and
every worker process adopts the resolved name — via the
``set_kv_cache_layout`` RPC, or ``KVCacheConfig.kv_cache_layout`` for
workers spawned after resolution — before the KV cache is allocated. Once
set the value is final and read with ``get_resolved_kv_cache_layout``,
which raises on ``None``. Tests and standalone tools may pre-set a value,
which resolution then honors as-is."""
prefix_match_unit: int | None = Field(default=None, gt=0)
"""The finest token boundary (in tokens) a prefix-cache hit can land on.
@@ -309,3 +340,12 @@ class CacheConfig:
str(cache_dtype),
)
return cache_dtype
def get_resolved_kv_cache_layout(self) -> KVCacheLayout:
if self.kv_cache_layout is None:
raise ValueError(
"KV cache layout has not been resolved yet; it is resolved once "
"by the engine core (resolve_kv_cache_layout) unless explicitly "
"set by the user."
)
return _layout_from_name(self.kv_cache_layout)