[DecodeBenchConnector] Fix HMA cache-group mapping (#54647)
Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
40b2f62061
commit
30dd1a7954
@@ -7,6 +7,8 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache
|
||||
with dummy values for decode performance benchmarking.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -22,6 +24,11 @@ from vllm.forward_context import ForwardContext
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .utils import (
|
||||
@@ -65,11 +72,10 @@ class DecodeBenchTestRunner:
|
||||
# Using simplified shape for testing
|
||||
num_heads = 4
|
||||
head_dim = 64
|
||||
layer_names = self.scheduler.kv_cache_config.kv_cache_groups[0].layer_names
|
||||
self.kv_caches = {
|
||||
f"layer_{i}": torch.zeros(
|
||||
num_gpu_blocks, 2, num_heads, block_size, head_dim
|
||||
)
|
||||
for i in range(2) # 2 layers for testing
|
||||
layer_name: torch.zeros(num_gpu_blocks, 2, num_heads, block_size, head_dim)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
|
||||
# Register KV caches with worker connector
|
||||
@@ -179,6 +185,97 @@ def test_decode_bench_connector_basic():
|
||||
assert torch.allclose(block_data, torch.tensor(0.015))
|
||||
|
||||
|
||||
def test_decode_bench_connector_fills_each_hma_group():
|
||||
"""Each cache group is filled using its own block IDs."""
|
||||
block_size = 16
|
||||
num_gpu_blocks = 8
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
)
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["group_0_layer"], kv_cache_spec),
|
||||
KVCacheGroupSpec(["group_1_layer"], kv_cache_spec),
|
||||
],
|
||||
)
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
kv_cache_config,
|
||||
)
|
||||
group_0_cache = torch.zeros(num_gpu_blocks, 2)
|
||||
group_1_cache = torch.zeros(num_gpu_blocks, 2)
|
||||
connector.register_kv_caches(
|
||||
{
|
||||
"group_0_layer": group_0_cache,
|
||||
"group_1_layer": group_1_cache,
|
||||
}
|
||||
)
|
||||
connector.bind_connector_metadata(
|
||||
DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([1], [5]), block_size)})
|
||||
)
|
||||
|
||||
connector.start_load_kv(
|
||||
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
)
|
||||
|
||||
assert torch.allclose(group_0_cache[1], torch.tensor(0.015))
|
||||
assert torch.count_nonzero(group_0_cache[5]) == 0
|
||||
assert torch.count_nonzero(group_1_cache[1]) == 0
|
||||
assert torch.allclose(group_1_cache[5], torch.tensor(0.015))
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_block_sizes():
|
||||
"""Scheduler selects only the blocks covering each group's token span."""
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=16,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
)
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
[f"group_{group_idx}_layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
for group_idx, block_size in enumerate((16, 32))
|
||||
]
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
),
|
||||
)
|
||||
request = MagicMock(request_id="request")
|
||||
blocks = MagicMock()
|
||||
blocks.get_block_ids.return_value = ([0, 1], [2, 3])
|
||||
|
||||
connector.update_state_after_alloc(request, blocks, num_external_tokens=17)
|
||||
metadata = connector.build_connector_meta(MagicMock())
|
||||
|
||||
assert isinstance(metadata, DecodeBenchConnectorMetadata)
|
||||
block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"]
|
||||
assert block_ids_per_group == ([0, 1], [2])
|
||||
assert num_tokens == 17
|
||||
|
||||
|
||||
def test_decode_bench_connector_no_refill():
|
||||
"""Test that DecodeBenchConnector only fills once per request."""
|
||||
block_size = 16
|
||||
|
||||
@@ -69,8 +69,8 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
# request_id -> (block_ids_per_group, num_tokens_to_fill)
|
||||
# block_ids_per_group is a tuple of lists, one per KV cache group
|
||||
# For standard attention: single group, e.g., ([1, 2, 3],)
|
||||
# For MLA: multiple groups, e.g., ([1, 2], [1, 2])
|
||||
# One group: ([1, 2, 3],)
|
||||
# Multiple groups: ([1, 2], [5, 6])
|
||||
reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]]
|
||||
|
||||
|
||||
@@ -95,9 +95,13 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.connector_worker: DecodeBenchConnectorWorker | None = None
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = DecodeBenchConnectorScheduler(vllm_config)
|
||||
self.connector_scheduler = DecodeBenchConnectorScheduler(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = DecodeBenchConnectorWorker(vllm_config)
|
||||
self.connector_worker = DecodeBenchConnectorWorker(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
@@ -182,9 +186,11 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
class DecodeBenchConnectorScheduler:
|
||||
"""Scheduler-side implementation for DecodeBenchConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig"):
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
self.vllm_config = vllm_config
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.group_block_sizes = tuple(
|
||||
group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
# Track which requests have already been filled
|
||||
self._filled_requests: set[str] = set()
|
||||
@@ -236,8 +242,7 @@ class DecodeBenchConnectorScheduler:
|
||||
Called after blocks are allocated. Store the block IDs so we can
|
||||
fill them with dummy values.
|
||||
|
||||
Supports both standard attention (single KV cache group) and MLA
|
||||
(multiple KV cache groups).
|
||||
Supports both single- and multi-group KV cache configurations.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
|
||||
@@ -246,18 +251,14 @@ class DecodeBenchConnectorScheduler:
|
||||
|
||||
# Get the block IDs that were allocated
|
||||
# block_groups is a tuple of lists, one per KV cache group
|
||||
# For standard attention: 1 group
|
||||
# For MLA: multiple groups (one per attention type)
|
||||
block_groups = blocks.get_block_ids()
|
||||
|
||||
# Calculate how many blocks we need to fill
|
||||
# num_external_tokens are the tokens we said we'd provide
|
||||
num_blocks_to_fill = cdiv(num_external_tokens, self.block_size)
|
||||
|
||||
# Extract the first num_blocks_to_fill blocks from each group
|
||||
# All groups should have the same block IDs for the same request
|
||||
# Extract the blocks covering num_external_tokens from each group
|
||||
block_ids_per_group = tuple(
|
||||
group_blocks[:num_blocks_to_fill] for group_blocks in block_groups
|
||||
group_blocks[: cdiv(num_external_tokens, group_block_size)]
|
||||
for group_blocks, group_block_size in zip(
|
||||
block_groups, self.group_block_sizes, strict=True
|
||||
)
|
||||
)
|
||||
|
||||
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
|
||||
@@ -269,9 +270,9 @@ class DecodeBenchConnectorScheduler:
|
||||
self._filled_requests.add(req_id)
|
||||
|
||||
logger.debug(
|
||||
"DecodeBenchConnector: Allocated %d blocks across %d KV cache groups "
|
||||
"DecodeBenchConnector: Allocated %s blocks across %d KV cache groups "
|
||||
"for request %s",
|
||||
num_blocks_to_fill,
|
||||
tuple(len(block_ids) for block_ids in block_ids_per_group),
|
||||
len(block_groups),
|
||||
req_id,
|
||||
)
|
||||
@@ -300,7 +301,7 @@ class DecodeBenchConnectorScheduler:
|
||||
class DecodeBenchConnectorWorker:
|
||||
"""Worker-side implementation for DecodeBenchConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig"):
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
self.vllm_config = vllm_config
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
|
||||
@@ -314,17 +315,15 @@ class DecodeBenchConnectorWorker:
|
||||
self.kv_caches: dict[str, torch.Tensor] | None = None
|
||||
|
||||
# Mapping from KV cache group index to list of layer names in that group
|
||||
self.group_to_layers: dict[int, list[str]] | None = None
|
||||
self.group_to_layers = {
|
||||
group_idx: list(group.layer_names)
|
||||
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
}
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Store references to the KV cache tensors and build group mapping."""
|
||||
"""Store references to the KV cache tensors."""
|
||||
self.kv_caches = kv_caches
|
||||
|
||||
# For simplicity, assume all layers belong to group 0 (standard attention)
|
||||
# For MLA models with multiple groups, the metadata will handle the mapping
|
||||
# We just need to fill the blocks specified in the metadata
|
||||
self.group_to_layers = {0: list(kv_caches.keys())}
|
||||
|
||||
logger.debug(
|
||||
"DecodeBenchConnector: Registered %d KV cache layers",
|
||||
len(kv_caches),
|
||||
@@ -337,13 +336,12 @@ class DecodeBenchConnectorWorker:
|
||||
This simulates having a populated KV cache from a prefill phase,
|
||||
allowing decode performance testing with larger context sizes.
|
||||
|
||||
Supports both standard attention (single group) and MLA (multiple groups).
|
||||
Supports both single- and multi-group KV cache configurations.
|
||||
"""
|
||||
if not metadata.reqs_to_fill:
|
||||
return
|
||||
|
||||
assert self.kv_caches is not None, "KV caches must be registered before filling"
|
||||
assert self.group_to_layers is not None, "Group mapping must be initialized"
|
||||
|
||||
for req_id, (block_ids_per_group, num_tokens) in metadata.reqs_to_fill.items():
|
||||
# Fill blocks for each KV cache group
|
||||
@@ -372,7 +370,6 @@ class DecodeBenchConnectorWorker:
|
||||
return
|
||||
|
||||
assert self.kv_caches is not None
|
||||
assert self.group_to_layers is not None
|
||||
|
||||
# Get the layers that belong to this group
|
||||
layer_names = self.group_to_layers.get(group_idx, [])
|
||||
|
||||
Reference in New Issue
Block a user