From e35298628fceb490d4ca2a21c0a6e8f63d3c6740 Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:05:58 -0400 Subject: [PATCH] [Bugfix][KV Connector] Fix DecodeBenchConnector prefix block selection (#54878) Signed-off-by: Jeff Ma Co-authored-by: OpenAI Codex --- .../unit/test_decode_bench_connector.py | 194 ++++++++++++++++-- .../kv_connector/v1/decode_bench_connector.py | 64 ++++-- 2 files changed, 230 insertions(+), 28 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index 66e403a31f..76bb4648a9 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -7,8 +7,6 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache with dummy values for decode performance benchmarking. """ -from unittest.mock import MagicMock - import pytest import torch @@ -24,9 +22,11 @@ from vllm.forward_context import ForwardContext from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.kv_cache_utils import ( + BlockHash, KVCacheBlock, get_request_block_hasher, init_none_hash, + make_block_hash_with_group_id, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler @@ -194,6 +194,43 @@ def test_decode_bench_connector_basic(): assert torch.allclose(block_data, torch.tensor(0.015)) +def test_decode_bench_connector_prefix_hit_fills_only_suffix_blocks(): + """A local prefix-cache hit shifts the fill to the suffix blocks.""" + block_size = 16 + runner = DecodeBenchTestRunner(block_size=block_size, num_gpu_blocks=100) + + prefix = list(range(1, 1 + block_size * 3)) + req1 = runner.new_request(prefix) + _, metadata = runner.run_single_step() + (prefix_block_ids,), _ = metadata.reqs_to_fill[req1.request_id] + assert len(prefix_block_ids) == 3 + + scheduler_output = runner.scheduler.schedule() + runner.scheduler.update_from_output( + scheduler_output, + create_model_runner_output( + reqs=runner.scheduler.running, + token_id=EOS_TOKEN_ID, + use_eos=True, + ), + ) + assert not runner.scheduler.running + + suffix = list(range(1000, 1000 + block_size * 2)) + req2 = runner.new_request(prefix + suffix) + _, metadata = runner.run_single_step() + (fill_block_ids,), num_tokens_to_fill = metadata.reqs_to_fill[req2.request_id] + + req2_block_ids = runner.scheduler.kv_cache_manager.get_block_ids(req2.request_id)[0] + assert req2_block_ids[:3] == prefix_block_ids + assert num_tokens_to_fill == len(suffix) - 1 + assert fill_block_ids == req2_block_ids[3:5] + assert set(fill_block_ids).isdisjoint(prefix_block_ids) + for kv_cache in runner.kv_caches.values(): + for block_id in fill_block_ids: + assert torch.allclose(kv_cache[block_id], torch.tensor(0.015)) + + def test_decode_bench_connector_fills_each_hma_group(): """Each cache group is filled using its own block IDs.""" block_size = 16 @@ -288,8 +325,8 @@ def test_decode_bench_connector_zero_fills_circular_buffer_groups(): assert torch.equal(circular_cache, expected_circular) -def test_decode_bench_connector_uses_per_group_block_sizes(): - """Scheduler selects only the blocks covering each group's token span.""" +def test_decode_bench_connector_selects_external_blocks_per_group(): + """External fills follow the local prefix using each group's block size.""" vllm_config = create_vllm_config( block_size=16, max_num_batched_tokens=1000, @@ -297,15 +334,23 @@ def test_decode_bench_connector_uses_per_group_block_sizes(): ) kv_cache_groups = [ KVCacheGroupSpec( - [f"group_{group_idx}_layer"], + ["full_attention"], FullAttentionSpec( - block_size=block_size, + block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32, ), - ) - for group_idx, block_size in enumerate((16, 32)) + ), + KVCacheGroupSpec( + ["compressor_ring"], + CircularBufferSpec( + block_size=32, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), ] connector = DecodeBenchConnector( vllm_config, @@ -316,17 +361,134 @@ def test_decode_bench_connector_uses_per_group_block_sizes(): kv_cache_groups=kv_cache_groups, ), ) - request = MagicMock(request_id="request") - blocks = MagicMock() - blocks.get_block_ids.return_value = ([0, 1], [2, 3]) + num_local_tokens = 32 + num_external_tokens = 17 + request = create_request( + request_id=1, + num_tokens=num_local_tokens + num_external_tokens + 1, + block_size=16, + ) + blocks = KVCacheBlocks( + ( + [ + KVCacheBlock(block_id=0), + KVCacheBlock(block_id=1), + # Full external blocks may already be hashed by allocate_slots(). + KVCacheBlock( + block_id=2, + _block_hash=make_block_hash_with_group_id( + BlockHash(b"external-full-block"), 0 + ), + _block_hash_num_tokens=48, + ), + KVCacheBlock(block_id=3), + ], + [KVCacheBlock(block_id=4)], + ) + ) - connector.update_state_after_alloc(request, blocks, num_external_tokens=17) - metadata = connector.build_connector_meta(MagicMock()) + connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = connector.build_connector_meta(SchedulerOutput.make_empty()) assert isinstance(metadata, DecodeBenchConnectorMetadata) - block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"] - assert block_ids_per_group == ([0, 1], [2]) - assert num_tokens == 17 + block_ids_per_group, num_tokens = metadata.reqs_to_fill[request.request_id] + assert block_ids_per_group == ([2, 3], [4]) + assert num_tokens == num_external_tokens + + +def test_decode_bench_connector_selects_unaligned_external_block(): + """An external interval includes the block containing its first token.""" + block_size = 32 + vllm_config = create_vllm_config( + block_size=block_size, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + ) + kv_cache_spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.SCHEDULER, + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], + ), + ) + num_local_tokens = 23 + num_external_tokens = 17 + request = create_request( + request_id=1, + num_tokens=num_local_tokens + num_external_tokens + 1, + block_size=block_size, + ) + blocks = KVCacheBlocks(([KVCacheBlock(block_id=1), KVCacheBlock(block_id=2)],)) + + connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = connector.build_connector_meta(SchedulerOutput.make_empty()) + + assert isinstance(metadata, DecodeBenchConnectorMetadata) + assert metadata.reqs_to_fill[request.request_id] == ( + ([1, 2],), + num_external_tokens, + ) + + +def test_decode_bench_connector_skips_null_padding(): + """External fills exclude null padding from a sliding-window group.""" + block_size = 16 + vllm_config = create_vllm_config( + block_size=block_size, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + ) + kv_cache_spec = SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=2 * block_size, + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.SCHEDULER, + KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], + ), + ) + num_external_tokens = 4 * block_size + request = create_request( + request_id=1, + num_tokens=num_external_tokens + 1, + block_size=block_size, + ) + null_block = KVCacheBlock(block_id=0, is_null=True) + blocks = KVCacheBlocks( + ( + [ + null_block, + null_block, + KVCacheBlock(block_id=1), + KVCacheBlock(block_id=2), + KVCacheBlock(block_id=3), + ], + ) + ) + + connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = connector.build_connector_meta(SchedulerOutput.make_empty()) + + assert isinstance(metadata, DecodeBenchConnectorMetadata) + assert metadata.reqs_to_fill[request.request_id] == ( + ([1, 2],), + num_external_tokens, + ) def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index b9f4cc15fe..83b468d265 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -193,6 +193,7 @@ class DecodeBenchConnectorScheduler: def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size + self.kv_cache_groups = kv_cache_config.kv_cache_groups self.group_block_sizes = tuple( resolve_dcp_kv_block_size( group.kv_cache_spec, @@ -201,7 +202,7 @@ class DecodeBenchConnectorScheduler: dcp_world_size, ), ) - for group in kv_cache_config.kv_cache_groups + for group in self.kv_cache_groups ) # Track which requests have already been filled @@ -261,17 +262,56 @@ class DecodeBenchConnectorScheduler: if num_external_tokens == 0: return - # Get the block IDs that were allocated - # block_groups is a tuple of lists, one per KV cache group - block_groups = blocks.get_block_ids() - - # Extract the blocks covering the external tokens from each group - block_ids_per_group = tuple( - group_blocks[: cdiv(num_external_tokens, group_block_size)] - for group_blocks, group_block_size in zip( - block_groups, self.group_block_sizes, strict=True + total_computed_tokens = request.num_tokens - 1 + num_local_computed_tokens = total_computed_tokens - num_external_tokens + block_ids_per_group_list: list[list[int]] = [] + for group_idx, ( + group, + group_blocks, + group_block_size, + ) in enumerate( + zip( + self.kv_cache_groups, + blocks.blocks, + self.group_block_sizes, + strict=True, ) - ) + ): + is_circular_buffer = all( + isinstance(spec, CircularBufferSpec) + for spec in iter_layer_specs(group.kv_cache_spec) + ) + if not is_circular_buffer: + num_computed_blocks = cdiv(total_computed_tokens, group_block_size) + external_block_start = num_local_computed_tokens // group_block_size + assert ( + 0 + <= external_block_start + <= num_computed_blocks + <= len(group_blocks) + ), ( + "DecodeBenchConnector block range exceeds allocated blocks: " + f"request={req_id}, group={group_idx}, " + f"range=[{external_block_start}, {num_computed_blocks}), " + f"allocated={len(group_blocks)}" + ) + selected_blocks = group_blocks[external_block_start:num_computed_blocks] + else: + selected_blocks = group_blocks + + block_ids = [ + block.block_id for block in selected_blocks if not block.is_null + ] + if not block_ids: + logger.warning( + "DecodeBenchConnector: No blocks selected for KV cache group " + "%d with %d external tokens for request %s", + group_idx, + num_external_tokens, + req_id, + ) + block_ids_per_group_list.append(block_ids) + block_ids_per_group = tuple(block_ids_per_group_list) # Store the blocks to fill for all group. _pending_fills doesn't need cleanup # as it's cleared after build_connector_meta @@ -286,7 +326,7 @@ class DecodeBenchConnectorScheduler: "DecodeBenchConnector: Selected %d total blocks across %d KV cache " "groups for request %s (per-group counts: %s)", sum(block_counts), - len(block_groups), + len(blocks.blocks), req_id, ", ".join(map(str, block_counts)), )