[Bugfix][KV Connector] Fix DecodeBench DCP block selection (#54679)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-09-01 20:31:30 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 16f16876fd
commit 18c53727ce
2 changed files with 90 additions and 13 deletions
@@ -22,18 +22,26 @@ from vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector import
)
from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
KVCacheBlock,
get_request_block_hasher,
init_none_hash,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
SlidingWindowSpec,
)
from vllm.v1.request import Request
from .utils import (
EOS_TOKEN_ID,
create_model_runner_output,
create_request,
create_scheduler,
create_vllm_config,
)
@@ -276,6 +284,63 @@ def test_decode_bench_connector_uses_per_group_block_sizes():
assert num_tokens == 17
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
"""DCP scales full-attention blocks but not sliding-window blocks."""
block_size = 16
dcp_world_size = 2
vllm_config = create_vllm_config(
block_size=block_size,
kv_connector="DecodeBenchConnector",
)
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
full_attention = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
sliding_window = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * block_size,
)
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["full_attention"], full_attention),
KVCacheGroupSpec(["sliding_window"], sliding_window),
],
),
)
num_external_tokens = block_size * dcp_world_size + 1
request = create_request(
request_id=1,
num_tokens=num_external_tokens + 1,
block_size=block_size,
)
blocks = KVCacheBlocks(
(
[KVCacheBlock(block_id=i) for i in (1, 2, 3)],
[KVCacheBlock(block_id=i) for i in (5, 6, 7, 8)],
)
)
connector.update_state_after_alloc(request, blocks, num_external_tokens)
metadata = connector.build_connector_meta(SchedulerOutput.make_empty())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
assert metadata.reqs_to_fill[request.request_id] == (
([1, 2], [5, 6, 7]),
num_external_tokens,
)
def test_decode_bench_connector_no_refill():
"""Test that DecodeBenchConnector only fills once per request."""
block_size = 16
@@ -47,6 +47,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
dcp_world_size_for_kv_cache_spec,
resolve_dcp_kv_block_size,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -187,9 +191,16 @@ class DecodeBenchConnectorScheduler:
"""Scheduler-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.group_block_sizes = tuple(
group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups
resolve_dcp_kv_block_size(
group.kv_cache_spec,
dcp_world_size_for_kv_cache_spec(
group.kv_cache_spec,
dcp_world_size,
),
)
for group in kv_cache_config.kv_cache_groups
)
# Track which requests have already been filled
@@ -253,7 +264,7 @@ class DecodeBenchConnectorScheduler:
# block_groups is a tuple of lists, one per KV cache group
block_groups = blocks.get_block_ids()
# Extract the blocks covering num_external_tokens from each group
# Extract the blocks covering the external tokens from each group
block_ids_per_group = tuple(
group_blocks[: cdiv(num_external_tokens, group_block_size)]
for group_blocks, group_block_size in zip(
@@ -269,12 +280,14 @@ class DecodeBenchConnectorScheduler:
)
self._filled_requests.add(req_id)
block_counts = tuple(len(group) for group in block_ids_per_group)
logger.debug(
"DecodeBenchConnector: Allocated %s blocks across %d KV cache groups "
"for request %s",
tuple(len(block_ids) for block_ids in block_ids_per_group),
"DecodeBenchConnector: Selected %d total blocks across %d KV cache "
"groups for request %s (per-group counts: %s)",
sum(block_counts),
len(block_groups),
req_id,
", ".join(map(str, block_counts)),
)
def build_connector_meta(
@@ -302,9 +315,6 @@ class DecodeBenchConnectorWorker:
"""Worker-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
# Get fill parameters from extra config
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
@@ -348,13 +358,15 @@ class DecodeBenchConnectorWorker:
for group_idx, block_ids in enumerate(block_ids_per_group):
self._fill_blocks(group_idx, block_ids, num_tokens)
block_counts = tuple(len(group) for group in block_ids_per_group)
logger.debug(
"DecodeBenchConnector: Filled %d blocks (%d tokens) across %d groups "
"for request %s",
len(block_ids_per_group[0]) if block_ids_per_group else 0,
"DecodeBenchConnector: Filled %d total blocks (%d tokens) across "
"%d groups for request %s (per-group counts: %s)",
sum(block_counts),
num_tokens,
len(block_ids_per_group),
req_id,
", ".join(map(str, block_counts)),
)
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):