From 18c53727cebb588771c5e2f62a207137b0d6dffd Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:31:30 -0400 Subject: [PATCH] [Bugfix][KV Connector] Fix DecodeBench DCP block selection (#54679) Signed-off-by: Jeff Ma Co-authored-by: OpenAI Codex --- .../unit/test_decode_bench_connector.py | 67 ++++++++++++++++++- .../kv_connector/v1/decode_bench_connector.py | 36 ++++++---- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index e249b5c5d6..6e7330606c 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -22,18 +22,26 @@ from vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector import ) from vllm.forward_context import ForwardContext from vllm.utils.hashing import sha256 -from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash +from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_utils import ( + KVCacheBlock, + get_request_block_hasher, + init_none_hash, +) +from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, + SlidingWindowSpec, ) from vllm.v1.request import Request from .utils import ( EOS_TOKEN_ID, create_model_runner_output, + create_request, create_scheduler, create_vllm_config, ) @@ -276,6 +284,63 @@ def test_decode_bench_connector_uses_per_group_block_sizes(): assert num_tokens == 17 +def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): + """DCP scales full-attention blocks but not sliding-window blocks.""" + block_size = 16 + dcp_world_size = 2 + vllm_config = create_vllm_config( + block_size=block_size, + kv_connector="DecodeBenchConnector", + ) + vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size + full_attention = FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ) + sliding_window = SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=2 * block_size, + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.SCHEDULER, + KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["full_attention"], full_attention), + KVCacheGroupSpec(["sliding_window"], sliding_window), + ], + ), + ) + num_external_tokens = block_size * dcp_world_size + 1 + request = create_request( + request_id=1, + num_tokens=num_external_tokens + 1, + block_size=block_size, + ) + blocks = KVCacheBlocks( + ( + [KVCacheBlock(block_id=i) for i in (1, 2, 3)], + [KVCacheBlock(block_id=i) for i in (5, 6, 7, 8)], + ) + ) + + connector.update_state_after_alloc(request, blocks, num_external_tokens) + metadata = connector.build_connector_meta(SchedulerOutput.make_empty()) + + assert isinstance(metadata, DecodeBenchConnectorMetadata) + assert metadata.reqs_to_fill[request.request_id] == ( + ([1, 2], [5, 6, 7]), + num_external_tokens, + ) + + def test_decode_bench_connector_no_refill(): """Test that DecodeBenchConnector only fills once per request.""" block_size = 16 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index 999e948eeb..c7c1078db1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -47,6 +47,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionMetadata +from vllm.v1.core.kv_cache_utils import ( + dcp_world_size_for_kv_cache_spec, + resolve_dcp_kv_block_size, +) if TYPE_CHECKING: from vllm.config import VllmConfig @@ -187,9 +191,16 @@ class DecodeBenchConnectorScheduler: """Scheduler-side implementation for DecodeBenchConnector.""" def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): - self.vllm_config = vllm_config + dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size self.group_block_sizes = tuple( - group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups + resolve_dcp_kv_block_size( + group.kv_cache_spec, + dcp_world_size_for_kv_cache_spec( + group.kv_cache_spec, + dcp_world_size, + ), + ) + for group in kv_cache_config.kv_cache_groups ) # Track which requests have already been filled @@ -253,7 +264,7 @@ class DecodeBenchConnectorScheduler: # block_groups is a tuple of lists, one per KV cache group block_groups = blocks.get_block_ids() - # Extract the blocks covering num_external_tokens from each group + # Extract the blocks covering the external tokens from each group block_ids_per_group = tuple( group_blocks[: cdiv(num_external_tokens, group_block_size)] for group_blocks, group_block_size in zip( @@ -269,12 +280,14 @@ class DecodeBenchConnectorScheduler: ) self._filled_requests.add(req_id) + block_counts = tuple(len(group) for group in block_ids_per_group) logger.debug( - "DecodeBenchConnector: Allocated %s blocks across %d KV cache groups " - "for request %s", - tuple(len(block_ids) for block_ids in block_ids_per_group), + "DecodeBenchConnector: Selected %d total blocks across %d KV cache " + "groups for request %s (per-group counts: %s)", + sum(block_counts), len(block_groups), req_id, + ", ".join(map(str, block_counts)), ) def build_connector_meta( @@ -302,9 +315,6 @@ class DecodeBenchConnectorWorker: """Worker-side implementation for DecodeBenchConnector.""" def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): - self.vllm_config = vllm_config - self.block_size = vllm_config.cache_config.block_size - # Get fill parameters from extra config kv_transfer_config = vllm_config.kv_transfer_config assert kv_transfer_config is not None @@ -348,13 +358,15 @@ class DecodeBenchConnectorWorker: for group_idx, block_ids in enumerate(block_ids_per_group): self._fill_blocks(group_idx, block_ids, num_tokens) + block_counts = tuple(len(group) for group in block_ids_per_group) logger.debug( - "DecodeBenchConnector: Filled %d blocks (%d tokens) across %d groups " - "for request %s", - len(block_ids_per_group[0]) if block_ids_per_group else 0, + "DecodeBenchConnector: Filled %d total blocks (%d tokens) across " + "%d groups for request %s (per-group counts: %s)", + sum(block_counts), num_tokens, len(block_ids_per_group), req_id, + ", ".join(map(str, block_counts)), ) def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):