[Bugfix][KV Connector] Fix DecodeBenchConnector prefix block selection (#54878)
Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
156050598e
commit
e35298628f
@@ -7,8 +7,6 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache
|
||||
with dummy values for decode performance benchmarking.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -24,9 +22,11 @@ from vllm.forward_context import ForwardContext
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
make_block_hash_with_group_id,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
@@ -194,6 +194,43 @@ def test_decode_bench_connector_basic():
|
||||
assert torch.allclose(block_data, torch.tensor(0.015))
|
||||
|
||||
|
||||
def test_decode_bench_connector_prefix_hit_fills_only_suffix_blocks():
|
||||
"""A local prefix-cache hit shifts the fill to the suffix blocks."""
|
||||
block_size = 16
|
||||
runner = DecodeBenchTestRunner(block_size=block_size, num_gpu_blocks=100)
|
||||
|
||||
prefix = list(range(1, 1 + block_size * 3))
|
||||
req1 = runner.new_request(prefix)
|
||||
_, metadata = runner.run_single_step()
|
||||
(prefix_block_ids,), _ = metadata.reqs_to_fill[req1.request_id]
|
||||
assert len(prefix_block_ids) == 3
|
||||
|
||||
scheduler_output = runner.scheduler.schedule()
|
||||
runner.scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=runner.scheduler.running,
|
||||
token_id=EOS_TOKEN_ID,
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
assert not runner.scheduler.running
|
||||
|
||||
suffix = list(range(1000, 1000 + block_size * 2))
|
||||
req2 = runner.new_request(prefix + suffix)
|
||||
_, metadata = runner.run_single_step()
|
||||
(fill_block_ids,), num_tokens_to_fill = metadata.reqs_to_fill[req2.request_id]
|
||||
|
||||
req2_block_ids = runner.scheduler.kv_cache_manager.get_block_ids(req2.request_id)[0]
|
||||
assert req2_block_ids[:3] == prefix_block_ids
|
||||
assert num_tokens_to_fill == len(suffix) - 1
|
||||
assert fill_block_ids == req2_block_ids[3:5]
|
||||
assert set(fill_block_ids).isdisjoint(prefix_block_ids)
|
||||
for kv_cache in runner.kv_caches.values():
|
||||
for block_id in fill_block_ids:
|
||||
assert torch.allclose(kv_cache[block_id], torch.tensor(0.015))
|
||||
|
||||
|
||||
def test_decode_bench_connector_fills_each_hma_group():
|
||||
"""Each cache group is filled using its own block IDs."""
|
||||
block_size = 16
|
||||
@@ -288,8 +325,8 @@ def test_decode_bench_connector_zero_fills_circular_buffer_groups():
|
||||
assert torch.equal(circular_cache, expected_circular)
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_block_sizes():
|
||||
"""Scheduler selects only the blocks covering each group's token span."""
|
||||
def test_decode_bench_connector_selects_external_blocks_per_group():
|
||||
"""External fills follow the local prefix using each group's block size."""
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=16,
|
||||
max_num_batched_tokens=1000,
|
||||
@@ -297,15 +334,23 @@ def test_decode_bench_connector_uses_per_group_block_sizes():
|
||||
)
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
[f"group_{group_idx}_layer"],
|
||||
["full_attention"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
for group_idx, block_size in enumerate((16, 32))
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["compressor_ring"],
|
||||
CircularBufferSpec(
|
||||
block_size=32,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
]
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
@@ -316,17 +361,134 @@ def test_decode_bench_connector_uses_per_group_block_sizes():
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
),
|
||||
)
|
||||
request = MagicMock(request_id="request")
|
||||
blocks = MagicMock()
|
||||
blocks.get_block_ids.return_value = ([0, 1], [2, 3])
|
||||
num_local_tokens = 32
|
||||
num_external_tokens = 17
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
num_tokens=num_local_tokens + num_external_tokens + 1,
|
||||
block_size=16,
|
||||
)
|
||||
blocks = KVCacheBlocks(
|
||||
(
|
||||
[
|
||||
KVCacheBlock(block_id=0),
|
||||
KVCacheBlock(block_id=1),
|
||||
# Full external blocks may already be hashed by allocate_slots().
|
||||
KVCacheBlock(
|
||||
block_id=2,
|
||||
_block_hash=make_block_hash_with_group_id(
|
||||
BlockHash(b"external-full-block"), 0
|
||||
),
|
||||
_block_hash_num_tokens=48,
|
||||
),
|
||||
KVCacheBlock(block_id=3),
|
||||
],
|
||||
[KVCacheBlock(block_id=4)],
|
||||
)
|
||||
)
|
||||
|
||||
connector.update_state_after_alloc(request, blocks, num_external_tokens=17)
|
||||
metadata = connector.build_connector_meta(MagicMock())
|
||||
connector.update_state_after_alloc(request, blocks, num_external_tokens)
|
||||
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
|
||||
|
||||
assert isinstance(metadata, DecodeBenchConnectorMetadata)
|
||||
block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"]
|
||||
assert block_ids_per_group == ([0, 1], [2])
|
||||
assert num_tokens == 17
|
||||
block_ids_per_group, num_tokens = metadata.reqs_to_fill[request.request_id]
|
||||
assert block_ids_per_group == ([2, 3], [4])
|
||||
assert num_tokens == num_external_tokens
|
||||
|
||||
|
||||
def test_decode_bench_connector_selects_unaligned_external_block():
|
||||
"""An external interval includes the block containing its first token."""
|
||||
block_size = 32
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
)
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
),
|
||||
)
|
||||
num_local_tokens = 23
|
||||
num_external_tokens = 17
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
num_tokens=num_local_tokens + num_external_tokens + 1,
|
||||
block_size=block_size,
|
||||
)
|
||||
blocks = KVCacheBlocks(([KVCacheBlock(block_id=1), KVCacheBlock(block_id=2)],))
|
||||
|
||||
connector.update_state_after_alloc(request, blocks, num_external_tokens)
|
||||
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
|
||||
|
||||
assert isinstance(metadata, DecodeBenchConnectorMetadata)
|
||||
assert metadata.reqs_to_fill[request.request_id] == (
|
||||
([1, 2],),
|
||||
num_external_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_decode_bench_connector_skips_null_padding():
|
||||
"""External fills exclude null padding from a sliding-window group."""
|
||||
block_size = 16
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
)
|
||||
kv_cache_spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=2 * block_size,
|
||||
)
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
),
|
||||
)
|
||||
num_external_tokens = 4 * block_size
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
num_tokens=num_external_tokens + 1,
|
||||
block_size=block_size,
|
||||
)
|
||||
null_block = KVCacheBlock(block_id=0, is_null=True)
|
||||
blocks = KVCacheBlocks(
|
||||
(
|
||||
[
|
||||
null_block,
|
||||
null_block,
|
||||
KVCacheBlock(block_id=1),
|
||||
KVCacheBlock(block_id=2),
|
||||
KVCacheBlock(block_id=3),
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
connector.update_state_after_alloc(request, blocks, num_external_tokens)
|
||||
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
|
||||
|
||||
assert isinstance(metadata, DecodeBenchConnectorMetadata)
|
||||
assert metadata.reqs_to_fill[request.request_id] == (
|
||||
([1, 2],),
|
||||
num_external_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
|
||||
@@ -193,6 +193,7 @@ class DecodeBenchConnectorScheduler:
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.kv_cache_groups = kv_cache_config.kv_cache_groups
|
||||
self.group_block_sizes = tuple(
|
||||
resolve_dcp_kv_block_size(
|
||||
group.kv_cache_spec,
|
||||
@@ -201,7 +202,7 @@ class DecodeBenchConnectorScheduler:
|
||||
dcp_world_size,
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for group in self.kv_cache_groups
|
||||
)
|
||||
|
||||
# Track which requests have already been filled
|
||||
@@ -261,17 +262,56 @@ class DecodeBenchConnectorScheduler:
|
||||
if num_external_tokens == 0:
|
||||
return
|
||||
|
||||
# Get the block IDs that were allocated
|
||||
# block_groups is a tuple of lists, one per KV cache group
|
||||
block_groups = blocks.get_block_ids()
|
||||
|
||||
# Extract the blocks covering the external tokens from each group
|
||||
block_ids_per_group = tuple(
|
||||
group_blocks[: cdiv(num_external_tokens, group_block_size)]
|
||||
for group_blocks, group_block_size in zip(
|
||||
block_groups, self.group_block_sizes, strict=True
|
||||
total_computed_tokens = request.num_tokens - 1
|
||||
num_local_computed_tokens = total_computed_tokens - num_external_tokens
|
||||
block_ids_per_group_list: list[list[int]] = []
|
||||
for group_idx, (
|
||||
group,
|
||||
group_blocks,
|
||||
group_block_size,
|
||||
) in enumerate(
|
||||
zip(
|
||||
self.kv_cache_groups,
|
||||
blocks.blocks,
|
||||
self.group_block_sizes,
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
):
|
||||
is_circular_buffer = all(
|
||||
isinstance(spec, CircularBufferSpec)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
if not is_circular_buffer:
|
||||
num_computed_blocks = cdiv(total_computed_tokens, group_block_size)
|
||||
external_block_start = num_local_computed_tokens // group_block_size
|
||||
assert (
|
||||
0
|
||||
<= external_block_start
|
||||
<= num_computed_blocks
|
||||
<= len(group_blocks)
|
||||
), (
|
||||
"DecodeBenchConnector block range exceeds allocated blocks: "
|
||||
f"request={req_id}, group={group_idx}, "
|
||||
f"range=[{external_block_start}, {num_computed_blocks}), "
|
||||
f"allocated={len(group_blocks)}"
|
||||
)
|
||||
selected_blocks = group_blocks[external_block_start:num_computed_blocks]
|
||||
else:
|
||||
selected_blocks = group_blocks
|
||||
|
||||
block_ids = [
|
||||
block.block_id for block in selected_blocks if not block.is_null
|
||||
]
|
||||
if not block_ids:
|
||||
logger.warning(
|
||||
"DecodeBenchConnector: No blocks selected for KV cache group "
|
||||
"%d with %d external tokens for request %s",
|
||||
group_idx,
|
||||
num_external_tokens,
|
||||
req_id,
|
||||
)
|
||||
block_ids_per_group_list.append(block_ids)
|
||||
block_ids_per_group = tuple(block_ids_per_group_list)
|
||||
|
||||
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
|
||||
# as it's cleared after build_connector_meta
|
||||
@@ -286,7 +326,7 @@ class DecodeBenchConnectorScheduler:
|
||||
"DecodeBenchConnector: Selected %d total blocks across %d KV cache "
|
||||
"groups for request %s (per-group counts: %s)",
|
||||
sum(block_counts),
|
||||
len(block_groups),
|
||||
len(blocks.blocks),
|
||||
req_id,
|
||||
", ".join(map(str, block_counts)),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user