[DecodeBenchConnector] Fix HMA cache-group mapping (#54647)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jeff (Junze) Ma
2026-09-01 05:52:50 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 40b2f62061
commit 30dd1a7954
2 changed files with 128 additions and 34 deletions
@@ -7,6 +7,8 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache
with dummy values for decode performance benchmarking.
"""
from unittest.mock import MagicMock
import pytest
import torch
@@ -22,6 +24,11 @@ from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
from vllm.v1.request import Request
from .utils import (
@@ -65,11 +72,10 @@ class DecodeBenchTestRunner:
# Using simplified shape for testing
num_heads = 4
head_dim = 64
layer_names = self.scheduler.kv_cache_config.kv_cache_groups[0].layer_names
self.kv_caches = {
f"layer_{i}": torch.zeros(
num_gpu_blocks, 2, num_heads, block_size, head_dim
)
for i in range(2) # 2 layers for testing
layer_name: torch.zeros(num_gpu_blocks, 2, num_heads, block_size, head_dim)
for layer_name in layer_names
}
# Register KV caches with worker connector
@@ -179,6 +185,97 @@ def test_decode_bench_connector_basic():
assert torch.allclose(block_data, torch.tensor(0.015))
def test_decode_bench_connector_fills_each_hma_group():
"""Each cache group is filled using its own block IDs."""
block_size = 16
num_gpu_blocks = 8
vllm_config = create_vllm_config(
block_size=block_size,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["group_0_layer"], kv_cache_spec),
KVCacheGroupSpec(["group_1_layer"], kv_cache_spec),
],
)
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.WORKER,
kv_cache_config,
)
group_0_cache = torch.zeros(num_gpu_blocks, 2)
group_1_cache = torch.zeros(num_gpu_blocks, 2)
connector.register_kv_caches(
{
"group_0_layer": group_0_cache,
"group_1_layer": group_1_cache,
}
)
connector.bind_connector_metadata(
DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([1], [5]), block_size)})
)
connector.start_load_kv(
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
)
assert torch.allclose(group_0_cache[1], torch.tensor(0.015))
assert torch.count_nonzero(group_0_cache[5]) == 0
assert torch.count_nonzero(group_1_cache[1]) == 0
assert torch.allclose(group_1_cache[5], torch.tensor(0.015))
def test_decode_bench_connector_uses_per_group_block_sizes():
"""Scheduler selects only the blocks covering each group's token span."""
vllm_config = create_vllm_config(
block_size=16,
max_num_batched_tokens=1000,
kv_connector="DecodeBenchConnector",
)
kv_cache_groups = [
KVCacheGroupSpec(
[f"group_{group_idx}_layer"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
for group_idx, block_size in enumerate((16, 32))
]
connector = DecodeBenchConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=kv_cache_groups,
),
)
request = MagicMock(request_id="request")
blocks = MagicMock()
blocks.get_block_ids.return_value = ([0, 1], [2, 3])
connector.update_state_after_alloc(request, blocks, num_external_tokens=17)
metadata = connector.build_connector_meta(MagicMock())
assert isinstance(metadata, DecodeBenchConnectorMetadata)
block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"]
assert block_ids_per_group == ([0, 1], [2])
assert num_tokens == 17
def test_decode_bench_connector_no_refill():
"""Test that DecodeBenchConnector only fills once per request."""
block_size = 16