[DecodeBenchConnector] Fix HMA cache-group mapping (#54647)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-09-01 05:52:50 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 40b2f62061
commit 30dd1a7954
2 changed files with 128 additions and 34 deletions
@@ -7,6 +7,8 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache
with dummy values for decode performance benchmarking.
"""
from unittest.mock import MagicMock
import pytest
import torch
@@ -22,6 +24,11 @@ from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
from vllm.v1.request import Request
from .utils import (
@@ -65,11 +72,10 @@ class DecodeBenchTestRunner:
# Using simplified shape for testing
num_heads = 4
head_dim = 64
layer_names = self.scheduler.kv_cache_config.kv_cache_groups[0].layer_names
self.kv_caches = {
f"layer_{i}": torch.zeros(
num_gpu_blocks, 2, num_heads, block_size, head_dim
)
for i in range(2) # 2 layers for testing
layer_name: torch.zeros(num_gpu_blocks, 2, num_heads, block_size, head_dim)
for layer_name in layer_names
}
# Register KV caches with worker connector
@@ -179,6 +185,97 @@ def test_decode_bench_connector_basic():
assert torch.allclose(block_data, torch.tensor(0.015))
def test_decode_bench_connector_fills_each_hma_group():
"""Each cache group is filled using its own block IDs."""
block_size = 16
num_gpu_blocks = 8
vllm_config = create_vllm_config(
block_size=block_size,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["group_0_layer"], kv_cache_spec),
KVCacheGroupSpec(["group_1_layer"], kv_cache_spec),
],
)
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.WORKER,
kv_cache_config,
)
group_0_cache = torch.zeros(num_gpu_blocks, 2)
group_1_cache = torch.zeros(num_gpu_blocks, 2)
connector.register_kv_caches(
{
"group_0_layer": group_0_cache,
"group_1_layer": group_1_cache,
}
)
connector.bind_connector_metadata(
DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([1], [5]), block_size)})
)
connector.start_load_kv(
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
)
assert torch.allclose(group_0_cache[1], torch.tensor(0.015))
assert torch.count_nonzero(group_0_cache[5]) == 0
assert torch.count_nonzero(group_1_cache[1]) == 0
assert torch.allclose(group_1_cache[5], torch.tensor(0.015))
def test_decode_bench_connector_uses_per_group_block_sizes():
"""Scheduler selects only the blocks covering each group's token span."""
vllm_config = create_vllm_config(
block_size=16,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_groups = [
KVCacheGroupSpec(
[f"group_{group_idx}_layer"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
for group_idx, block_size in enumerate((16, 32))
]
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=kv_cache_groups,
),
)
request = MagicMock(request_id="request")
blocks = MagicMock()
blocks.get_block_ids.return_value = ([0, 1], [2, 3])
connector.update_state_after_alloc(request, blocks, num_external_tokens=17)
metadata = connector.build_connector_meta(MagicMock())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"]
assert block_ids_per_group == ([0, 1], [2])
assert num_tokens == 17
def test_decode_bench_connector_no_refill():
"""Test that DecodeBenchConnector only fills once per request."""
block_size = 16
@@ -69,8 +69,8 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
# request_id -> (block_ids_per_group, num_tokens_to_fill)
# block_ids_per_group is a tuple of lists, one per KV cache group
# For standard attention: single group, e.g., ([1, 2, 3],)
# For MLA: multiple groups, e.g., ([1, 2], [1, 2])
# One group: ([1, 2, 3],)
# Multiple groups: ([1, 2], [5, 6])
reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]]
@@ -95,9 +95,13 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_worker: DecodeBenchConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = DecodeBenchConnectorScheduler(vllm_config)
self.connector_scheduler = DecodeBenchConnectorScheduler(
vllm_config, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = DecodeBenchConnectorWorker(vllm_config)
self.connector_worker = DecodeBenchConnectorWorker(
vllm_config, kv_cache_config
)
# ==============================
# Worker-side methods
@@ -182,9 +186,11 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
class DecodeBenchConnectorScheduler:
"""Scheduler-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig"):
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
self.group_block_sizes = tuple(
group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups
)
# Track which requests have already been filled
self._filled_requests: set[str] = set()
@@ -236,8 +242,7 @@ class DecodeBenchConnectorScheduler:
Called after blocks are allocated. Store the block IDs so we can
fill them with dummy values.
Supports both standard attention (single KV cache group) and MLA
(multiple KV cache groups).
Supports both single- and multi-group KV cache configurations.
"""
req_id = request.request_id
@@ -246,18 +251,14 @@ class DecodeBenchConnectorScheduler:
# Get the block IDs that were allocated
# block_groups is a tuple of lists, one per KV cache group
# For standard attention: 1 group
# For MLA: multiple groups (one per attention type)
block_groups = blocks.get_block_ids()
# Calculate how many blocks we need to fill
# num_external_tokens are the tokens we said we'd provide
num_blocks_to_fill = cdiv(num_external_tokens, self.block_size)
# Extract the first num_blocks_to_fill blocks from each group
# All groups should have the same block IDs for the same request
# Extract the blocks covering num_external_tokens from each group
block_ids_per_group = tuple(
group_blocks[:num_blocks_to_fill] for group_blocks in block_groups
group_blocks[: cdiv(num_external_tokens, group_block_size)]
for group_blocks, group_block_size in zip(
block_groups, self.group_block_sizes, strict=True
)
)
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
@@ -269,9 +270,9 @@ class DecodeBenchConnectorScheduler:
self._filled_requests.add(req_id)
logger.debug(
"DecodeBenchConnector: Allocated %d blocks across %d KV cache groups "
"DecodeBenchConnector: Allocated %s blocks across %d KV cache groups "
"for request %s",
num_blocks_to_fill,
tuple(len(block_ids) for block_ids in block_ids_per_group),
len(block_groups),
req_id,
)
@@ -300,7 +301,7 @@ class DecodeBenchConnectorScheduler:
class DecodeBenchConnectorWorker:
"""Worker-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig"):
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
self.vllm_config = vllm_config
self.block_size = vllm_config.cache_config.block_size
@@ -314,17 +315,15 @@ class DecodeBenchConnectorWorker:
self.kv_caches: dict[str, torch.Tensor] | None = None
# Mapping from KV cache group index to list of layer names in that group
self.group_to_layers: dict[int, list[str]] | None = None
self.group_to_layers = {
group_idx: list(group.layer_names)
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
}
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Store references to the KV cache tensors and build group mapping."""
"""Store references to the KV cache tensors."""
self.kv_caches = kv_caches
# For simplicity, assume all layers belong to group 0 (standard attention)
# For MLA models with multiple groups, the metadata will handle the mapping
# We just need to fill the blocks specified in the metadata
self.group_to_layers = {0: list(kv_caches.keys())}
logger.debug(
"DecodeBenchConnector: Registered %d KV cache layers",
len(kv_caches),
@@ -337,13 +336,12 @@ class DecodeBenchConnectorWorker:
This simulates having a populated KV cache from a prefill phase,
allowing decode performance testing with larger context sizes.
Supports both standard attention (single group) and MLA (multiple groups).
Supports both single- and multi-group KV cache configurations.
"""
if not metadata.reqs_to_fill:
return
assert self.kv_caches is not None, "KV caches must be registered before filling"
assert self.group_to_layers is not None, "Group mapping must be initialized"
for req_id, (block_ids_per_group, num_tokens) in metadata.reqs_to_fill.items():
# Fill blocks for each KV cache group
@@ -372,7 +370,6 @@ class DecodeBenchConnectorWorker:
return
assert self.kv_caches is not None
assert self.group_to_layers is not None
# Get the layers that belong to this group
layer_names = self.group_to_layers.get(group_idx, [])