[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-08-14 09:47:55 -04:00
committed by GitHub
co-authored by Claude
parent 63a9a5010a
commit 57bd0ed441
18 changed files with 204 additions and 219 deletions
+13 -13
View File
@@ -687,13 +687,15 @@ class Platform:
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
"""Bytes one token occupies in one layer, for the given dtype."""
return FullAttentionSpec(
spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=dtype,
kv_quant_mode=get_kv_quant_mode(cache_dtype),
).page_size_bytes
)
# The backend owns its packing
return backend_cls.customize_spec(spec).page_size_bytes
primary_dtype = (
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
@@ -809,23 +811,18 @@ class Platform:
# when all attention layers are TQ. With mixed skip+TQ the skip
# layers still use the standard layout — take max so mamba
# padding covers the largest actual page.
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_cfg = TurboQuantConfig.from_cache_dtype(
cache_config.cache_dtype, model_config.get_head_size()
)
tq_page = TQFullAttentionSpec(
tq_spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
head_size_v=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
tq_slot_size=tq_cfg.slot_size_aligned,
).page_size_bytes
)
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
if cache_config.kv_cache_dtype_skip_layers:
skip_page = FullAttentionSpec(
block_size=1,
@@ -840,12 +837,15 @@ class Platform:
else:
attn_page_size_1_token = tq_page
else:
attn_page_size_1_token = FullAttentionSpec(
attn_spec = FullAttentionSpec(
block_size=1,
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
)
attn_page_size_1_token = backend_cls.customize_spec(
attn_spec
).page_size_bytes
# Compute mamba page size