[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+102 -10
View File
@@ -15,7 +15,11 @@ import pytest
import torch
import vllm.v1.hisparse.binding as attn_utils_module
from tests.v1.attention.utils import dense_kv_cache_views
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -42,6 +46,8 @@ from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
)
@@ -79,7 +85,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
backend = SimpleNamespace(
get_name=lambda name=name: name,
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
get_supported_kernel_block_sizes=lambda _=None, sizes=sizes: sizes,
)
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
@@ -99,7 +105,8 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
resolved = attn_utils.get_kv_cache_spec(config)
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
assert resolved["dense"] is specs["dense"]
dense_backend = layers["dense"].get_attn_backend()
assert resolved["dense"] == customize_attention_spec(dense_backend, specs["dense"])
assert all(spec.block_size == block_size for spec in specs.values())
@@ -495,20 +502,17 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
@pytest.mark.parametrize(
(
"kernel_block_sizes",
"storage_block_size",
"expected_num_blocks",
"expected_num_states",
),
[
(None, None, 4, 64),
([256], None, 4, 64),
([64], None, 16, 16),
([64], 256, 4, 64),
(None, 4, 64),
([256], 4, 64),
([64], 16, 16),
],
)
def test_allocate_compressed_mla_cache(
kernel_block_sizes: list[int] | None,
storage_block_size: int | None,
expected_num_blocks: int,
expected_num_states: int,
):
@@ -518,7 +522,6 @@ def test_allocate_compressed_mla_cache(
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=4,
storage_block_size=storage_block_size,
)
num_pages = 4
config = KVCacheConfig(
@@ -744,3 +747,92 @@ def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
backing.untyped_storage().data_ptr()
== caches["indexer"].untyped_storage().data_ptr()
)
class _TableBuilder:
requires_block_table_width = False
supports_update_block_table = False
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
self.kv_cache_spec = kv_cache_spec
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
return common_attn_metadata.block_table_tensor
_VLLM_CONFIG = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=8192),
scheduler_config=SimpleNamespace(max_num_seqs=4),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
def _packed_group(tokens_per_state, row_bytes, supported_sizes):
spec = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=128,
dtype=torch.uint8,
state_content_bytes=row_bytes,
tokens_per_state=tokens_per_state,
)
backend = SimpleNamespace(
get_builder_cls=lambda: _TableBuilder,
get_supported_kernel_block_sizes=lambda _=None: supported_sizes,
get_name=lambda: "FAKE",
customize_spec=lambda spec: spec,
)
return AttentionGroup(backend, ["layer.0"], spec, 0)
def test_packed_compressed_group_gets_kernel_block_table():
"""A packed indexer cache whose kernel needs 64-state pages gets a table and
a bound view in kernel blocks that address the same packed-block rows."""
row, stride_rows = 132, 5 * 64
group = _packed_group(4, row, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
assert group.get_metadata_builder().kv_cache_spec.block_size == 256
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
manager = backing.view(3, 1, stride_rows, row)[:, :, :256]
pages = map_kv_caches_to_kernel_blocks({"layer.0": manager}, [group])["layer.0"]
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 10, 11, 12, 13]]
assert torch.equal(pages[table[0, 4:], 0].flatten(0, 1), manager[2, 0])
def test_split_group_maps_manager_block_table():
"""Groups split in a dense layout keep manager block tables; builders get
b * blocks_per_kv_block + j against the view allocated in kernel blocks."""
group = _packed_group(4, 132, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=256)
kernel_view = torch.zeros(12, 1, 64, 132)
bound = map_kv_caches_to_kernel_blocks({"layer.0": kernel_view}, [group])
assert bound["layer.0"] is kernel_view
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 8, 9, 10, 11]]
@pytest.mark.parametrize(
("block_size", "page_states"), [(1024, 64), (384, 32), (256, 0)]
)
def test_split_blocks_align_to_kernel_blocks(block_size, page_states):
"""Backends taking only fixed kernel blocks align packed blocks to the kernel
block the final block size is split into, and not at all when unsplit."""
group = _packed_group(4, 132, [128, 256])
spec = customize_attention_spec(group.backend, group.kv_cache_spec)
spec = spec.copy_with_new_block_size(block_size)
assert spec.get_block_stride_alignment() == max(page_states * 132, 1)
def test_packed_token_cache_cannot_be_split():
group = _packed_group(1, 1152, [64])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
packed = torch.zeros(3, 1, 2048, 1152, dtype=torch.uint8)[:, :, :1024]
with pytest.raises(ValueError, match="cannot be split"):
map_kv_caches_to_kernel_blocks({"layer.0": packed}, [group])