[5/N][KV-Cache Layout Refactor] Backend-published KV packing via customize_spec (#51704)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
63a9a5010a
commit
57bd0ed441
+13
-13
@@ -687,13 +687,15 @@ class Platform:
|
||||
|
||||
def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int:
|
||||
"""Bytes one token occupies in one layer, for the given dtype."""
|
||||
return FullAttentionSpec(
|
||||
spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(cache_dtype),
|
||||
).page_size_bytes
|
||||
)
|
||||
# The backend owns its packing
|
||||
return backend_cls.customize_spec(spec).page_size_bytes
|
||||
|
||||
primary_dtype = (
|
||||
STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
@@ -809,23 +811,18 @@ class Platform:
|
||||
# when all attention layers are TQ. With mixed skip+TQ the skip
|
||||
# layers still use the standard layout — take max so mamba
|
||||
# padding covers the largest actual page.
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_cfg = TurboQuantConfig.from_cache_dtype(
|
||||
cache_config.cache_dtype, model_config.get_head_size()
|
||||
)
|
||||
tq_page = TQFullAttentionSpec(
|
||||
tq_spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
head_size_v=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
tq_slot_size=tq_cfg.slot_size_aligned,
|
||||
).page_size_bytes
|
||||
)
|
||||
tq_page = TurboQuantAttentionBackend.customize_spec(tq_spec).page_size_bytes
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
skip_page = FullAttentionSpec(
|
||||
block_size=1,
|
||||
@@ -840,12 +837,15 @@ class Platform:
|
||||
else:
|
||||
attn_page_size_1_token = tq_page
|
||||
else:
|
||||
attn_page_size_1_token = FullAttentionSpec(
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
attn_page_size_1_token = backend_cls.customize_spec(
|
||||
attn_spec
|
||||
).page_size_bytes
|
||||
|
||||
# Compute mamba page size
|
||||
|
||||
Reference in New Issue
Block a user