[Bugfix][KV Connector] Fix DecodeBenchConnector prefix block selection (#54878)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-09-04 04:05:58 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 156050598e
commit e35298628f
2 changed files with 230 additions and 28 deletions
@@ -7,8 +7,6 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache
with dummy values for decode performance benchmarking.
"""
from unittest.mock import MagicMock
import pytest
import torch
@@ -24,9 +22,11 @@ from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
get_request_block_hasher,
init_none_hash,
make_block_hash_with_group_id,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
@@ -194,6 +194,43 @@ def test_decode_bench_connector_basic():
assert torch.allclose(block_data, torch.tensor(0.015))
def test_decode_bench_connector_prefix_hit_fills_only_suffix_blocks():
"""A local prefix-cache hit shifts the fill to the suffix blocks."""
block_size = 16
runner = DecodeBenchTestRunner(block_size=block_size, num_gpu_blocks=100)
prefix = list(range(1, 1 + block_size * 3))
req1 = runner.new_request(prefix)
_, metadata = runner.run_single_step()
(prefix_block_ids,), _ = metadata.reqs_to_fill[req1.request_id]
assert len(prefix_block_ids) == 3
scheduler_output = runner.scheduler.schedule()
runner.scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=runner.scheduler.running,
token_id=EOS_TOKEN_ID,
use_eos=True,
),
)
assert not runner.scheduler.running
suffix = list(range(1000, 1000 + block_size * 2))
req2 = runner.new_request(prefix + suffix)
_, metadata = runner.run_single_step()
(fill_block_ids,), num_tokens_to_fill = metadata.reqs_to_fill[req2.request_id]
req2_block_ids = runner.scheduler.kv_cache_manager.get_block_ids(req2.request_id)[0]
assert req2_block_ids[:3] == prefix_block_ids
assert num_tokens_to_fill == len(suffix) - 1
assert fill_block_ids == req2_block_ids[3:5]
assert set(fill_block_ids).isdisjoint(prefix_block_ids)
for kv_cache in runner.kv_caches.values():
for block_id in fill_block_ids:
assert torch.allclose(kv_cache[block_id], torch.tensor(0.015))
def test_decode_bench_connector_fills_each_hma_group():
"""Each cache group is filled using its own block IDs."""
block_size = 16
@@ -288,8 +325,8 @@ def test_decode_bench_connector_zero_fills_circular_buffer_groups():
assert torch.equal(circular_cache, expected_circular)
def test_decode_bench_connector_uses_per_group_block_sizes():
"""Scheduler selects only the blocks covering each group's token span."""
def test_decode_bench_connector_selects_external_blocks_per_group():
"""External fills follow the local prefix using each group's block size."""
vllm_config = create_vllm_config(
block_size=16,
max_num_batched_tokens=1000,
@@ -297,15 +334,23 @@ def test_decode_bench_connector_uses_per_group_block_sizes():
)
kv_cache_groups = [
KVCacheGroupSpec(
[f"group_{group_idx}_layer"],
["full_attention"],
FullAttentionSpec(
block_size=block_size,
block_size=16,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
for group_idx, block_size in enumerate((16, 32))
),
KVCacheGroupSpec(
["compressor_ring"],
CircularBufferSpec(
block_size=32,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
]
connector = DecodeBenchConnector(
vllm_config,
@@ -316,17 +361,134 @@ def test_decode_bench_connector_uses_per_group_block_sizes():
kv_cache_groups=kv_cache_groups,
),
)
request = MagicMock(request_id="request")
blocks = MagicMock()
blocks.get_block_ids.return_value = ([0, 1], [2, 3])
num_local_tokens = 32
num_external_tokens = 17
request = create_request(
request_id=1,
num_tokens=num_local_tokens + num_external_tokens + 1,
block_size=16,
)
blocks = KVCacheBlocks(
(
[
KVCacheBlock(block_id=0),
KVCacheBlock(block_id=1),
# Full external blocks may already be hashed by allocate_slots().
KVCacheBlock(
block_id=2,
_block_hash=make_block_hash_with_group_id(
BlockHash(b"external-full-block"), 0
),
_block_hash_num_tokens=48,
),
KVCacheBlock(block_id=3),
],
[KVCacheBlock(block_id=4)],
)
)
connector.update_state_after_alloc(request, blocks, num_external_tokens=17)
metadata = connector.build_connector_meta(MagicMock())
connector.update_state_after_alloc(request, blocks, num_external_tokens)
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"]
assert block_ids_per_group == ([0, 1], [2])
assert num_tokens == 17
block_ids_per_group, num_tokens = metadata.reqs_to_fill[request.request_id]
assert block_ids_per_group == ([2, 3], [4])
assert num_tokens == num_external_tokens
def test_decode_bench_connector_selects_unaligned_external_block():
"""An external interval includes the block containing its first token."""
block_size = 32
vllm_config = create_vllm_config(
block_size=block_size,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
),
)
num_local_tokens = 23
num_external_tokens = 17
request = create_request(
request_id=1,
num_tokens=num_local_tokens + num_external_tokens + 1,
block_size=block_size,
)
blocks = KVCacheBlocks(([KVCacheBlock(block_id=1), KVCacheBlock(block_id=2)],))
connector.update_state_after_alloc(request, blocks, num_external_tokens)
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
assert metadata.reqs_to_fill[request.request_id] == (
([1, 2],),
num_external_tokens,
)
def test_decode_bench_connector_skips_null_padding():
"""External fills exclude null padding from a sliding-window group."""
block_size = 16
vllm_config = create_vllm_config(
block_size=block_size,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * block_size,
)
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
),
)
num_external_tokens = 4 * block_size
request = create_request(
request_id=1,
num_tokens=num_external_tokens + 1,
block_size=block_size,
)
null_block = KVCacheBlock(block_id=0, is_null=True)
blocks = KVCacheBlocks(
(
[
null_block,
null_block,
KVCacheBlock(block_id=1),
KVCacheBlock(block_id=2),
KVCacheBlock(block_id=3),
],
)
)
connector.update_state_after_alloc(request, blocks, num_external_tokens)
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
assert metadata.reqs_to_fill[request.request_id] == (
([1, 2],),
num_external_tokens,
)
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
@@ -193,6 +193,7 @@ class DecodeBenchConnectorScheduler:
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
resolve_dcp_kv_block_size(
group.kv_cache_spec,
@@ -201,7 +202,7 @@ class DecodeBenchConnectorScheduler:
dcp_world_size,
),
)
for group in kv_cache_config.kv_cache_groups
for group in self.kv_cache_groups
)
# Track which requests have already been filled
@@ -261,17 +262,56 @@ class DecodeBenchConnectorScheduler:
if num_external_tokens == 0:
return
# Get the block IDs that were allocated
# block_groups is a tuple of lists, one per KV cache group
block_groups = blocks.get_block_ids()
# Extract the blocks covering the external tokens from each group
block_ids_per_group = tuple(
group_blocks[: cdiv(num_external_tokens, group_block_size)]
for group_blocks, group_block_size in zip(
block_groups, self.group_block_sizes, strict=True
total_computed_tokens = request.num_tokens - 1
num_local_computed_tokens = total_computed_tokens - num_external_tokens
block_ids_per_group_list: list[list[int]] = []
for group_idx, (
group,
group_blocks,
group_block_size,
) in enumerate(
zip(
self.kv_cache_groups,
blocks.blocks,
self.group_block_sizes,
strict=True,
)
)
):
is_circular_buffer = all(
isinstance(spec, CircularBufferSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
)
if not is_circular_buffer:
num_computed_blocks = cdiv(total_computed_tokens, group_block_size)
external_block_start = num_local_computed_tokens // group_block_size
assert (
0
<= external_block_start
<= num_computed_blocks
<= len(group_blocks)
), (
"DecodeBenchConnector block range exceeds allocated blocks: "
f"request={req_id}, group={group_idx}, "
f"range=[{external_block_start}, {num_computed_blocks}), "
f"allocated={len(group_blocks)}"
)
selected_blocks = group_blocks[external_block_start:num_computed_blocks]
else:
selected_blocks = group_blocks
block_ids = [
block.block_id for block in selected_blocks if not block.is_null
]
if not block_ids:
logger.warning(
"DecodeBenchConnector: No blocks selected for KV cache group "
"%d with %d external tokens for request %s",
group_idx,
num_external_tokens,
req_id,
)
block_ids_per_group_list.append(block_ids)
block_ids_per_group = tuple(block_ids_per_group_list)
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
# as it's cleared after build_connector_meta
@@ -286,7 +326,7 @@ class DecodeBenchConnectorScheduler:
"DecodeBenchConnector: Selected %d total blocks across %d KV cache "
"groups for request %s (per-group counts: %s)",
sum(block_counts),
len(block_groups),
len(blocks.blocks),
req_id,
", ".join(map(str, block_counts)),
)