[DecodeBenchConnector] Fix HMA cache-group mapping (#54647)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-09-01 05:52:50 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 40b2f62061
commit 30dd1a7954
2 changed files with 128 additions and 34 deletions
@@ -69,8 +69,8 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
# request_id -> (block_ids_per_group, num_tokens_to_fill)
# block_ids_per_group is a tuple of lists, one per KV cache group
# For standard attention: single group, e.g., ([1, 2, 3],)
# For MLA: multiple groups, e.g., ([1, 2], [1, 2])
# One group: ([1, 2, 3],)
# Multiple groups: ([1, 2], [5, 6])
reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]]
@@ -95,9 +95,13 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_worker: DecodeBenchConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = DecodeBenchConnectorScheduler(vllm_config)
self.connector_scheduler = DecodeBenchConnectorScheduler(
vllm_config, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = DecodeBenchConnectorWorker(vllm_config)
self.connector_worker = DecodeBenchConnectorWorker(
vllm_config, kv_cache_config
)
# ==============================
# Worker-side methods
@@ -182,9 +186,11 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
class DecodeBenchConnectorScheduler:
"""Scheduler-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig"):
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
self.group_block_sizes = tuple(
group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups
)
# Track which requests have already been filled
self._filled_requests: set[str] = set()
@@ -236,8 +242,7 @@ class DecodeBenchConnectorScheduler:
Called after blocks are allocated. Store the block IDs so we can
fill them with dummy values.
Supports both standard attention (single KV cache group) and MLA
(multiple KV cache groups).
Supports both single- and multi-group KV cache configurations.
"""
req_id = request.request_id
@@ -246,18 +251,14 @@ class DecodeBenchConnectorScheduler:
# Get the block IDs that were allocated
# block_groups is a tuple of lists, one per KV cache group
# For standard attention: 1 group
# For MLA: multiple groups (one per attention type)
block_groups = blocks.get_block_ids()
# Calculate how many blocks we need to fill
# num_external_tokens are the tokens we said we'd provide
num_blocks_to_fill = cdiv(num_external_tokens, self.block_size)
# Extract the first num_blocks_to_fill blocks from each group
# All groups should have the same block IDs for the same request
# Extract the blocks covering num_external_tokens from each group
block_ids_per_group = tuple(
group_blocks[:num_blocks_to_fill] for group_blocks in block_groups
group_blocks[: cdiv(num_external_tokens, group_block_size)]
for group_blocks, group_block_size in zip(
block_groups, self.group_block_sizes, strict=True
)
)
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
@@ -269,9 +270,9 @@ class DecodeBenchConnectorScheduler:
self._filled_requests.add(req_id)
logger.debug(
"DecodeBenchConnector: Allocated %d blocks across %d KV cache groups "
"DecodeBenchConnector: Allocated %s blocks across %d KV cache groups "
"for request %s",
num_blocks_to_fill,
tuple(len(block_ids) for block_ids in block_ids_per_group),
len(block_groups),
req_id,
)
@@ -300,7 +301,7 @@ class DecodeBenchConnectorScheduler:
class DecodeBenchConnectorWorker:
"""Worker-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig"):
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
@@ -314,17 +315,15 @@ class DecodeBenchConnectorWorker:
self.kv_caches: dict[str, torch.Tensor] | None = None
# Mapping from KV cache group index to list of layer names in that group
self.group_to_layers: dict[int, list[str]] | None = None
self.group_to_layers = {
group_idx: list(group.layer_names)
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
}
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Store references to the KV cache tensors and build group mapping."""
"""Store references to the KV cache tensors."""
self.kv_caches = kv_caches
# For simplicity, assume all layers belong to group 0 (standard attention)
# For MLA models with multiple groups, the metadata will handle the mapping
# We just need to fill the blocks specified in the metadata
self.group_to_layers = {0: list(kv_caches.keys())}
logger.debug(
"DecodeBenchConnector: Registered %d KV cache layers",
len(kv_caches),
@@ -337,13 +336,12 @@ class DecodeBenchConnectorWorker:
This simulates having a populated KV cache from a prefill phase,
allowing decode performance testing with larger context sizes.
Supports both standard attention (single group) and MLA (multiple groups).
Supports both single- and multi-group KV cache configurations.
"""
if not metadata.reqs_to_fill:
return
assert self.kv_caches is not None, "KV caches must be registered before filling"
assert self.group_to_layers is not None, "Group mapping must be initialized"
for req_id, (block_ids_per_group, num_tokens) in metadata.reqs_to_fill.items():
# Fill blocks for each KV cache group
@@ -372,7 +370,6 @@ class DecodeBenchConnectorWorker:
return
assert self.kv_caches is not None
assert self.group_to_layers is not None
# Get the layers that belong to this group
layer_names = self.group_to_layers.get(group_idx, [])