From 30dd1a7954ff91ae0359806ec56b22b6be0c529f Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:52:50 -0400 Subject: [PATCH] [DecodeBenchConnector] Fix HMA cache-group mapping (#54647) Signed-off-by: Jeff Ma Co-authored-by: OpenAI Codex --- .../unit/test_decode_bench_connector.py | 105 +++++++++++++++++- .../kv_connector/v1/decode_bench_connector.py | 57 +++++----- 2 files changed, 128 insertions(+), 34 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index 5f7c5eeefa..e249b5c5d6 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -7,6 +7,8 @@ Tests the functionality of the DecodeBenchConnector which fills KV cache with dummy values for decode performance benchmarking. """ +from unittest.mock import MagicMock + import pytest import torch @@ -22,6 +24,11 @@ from vllm.forward_context import ForwardContext from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, +) from vllm.v1.request import Request from .utils import ( @@ -65,11 +72,10 @@ class DecodeBenchTestRunner: # Using simplified shape for testing num_heads = 4 head_dim = 64 + layer_names = self.scheduler.kv_cache_config.kv_cache_groups[0].layer_names self.kv_caches = { - f"layer_{i}": torch.zeros( - num_gpu_blocks, 2, num_heads, block_size, head_dim - ) - for i in range(2) # 2 layers for testing + layer_name: torch.zeros(num_gpu_blocks, 2, num_heads, block_size, head_dim) + for layer_name in layer_names } # Register KV caches with worker connector @@ -179,6 +185,97 @@ def test_decode_bench_connector_basic(): assert torch.allclose(block_data, torch.tensor(0.015)) +def test_decode_bench_connector_fills_each_hma_group(): + """Each cache group is filled using its own block IDs.""" + block_size = 16 + num_gpu_blocks = 8 + vllm_config = create_vllm_config( + block_size=block_size, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + ) + kv_cache_spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ) + kv_cache_config = KVCacheConfig( + num_blocks=num_gpu_blocks, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["group_0_layer"], kv_cache_spec), + KVCacheGroupSpec(["group_1_layer"], kv_cache_spec), + ], + ) + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.WORKER, + kv_cache_config, + ) + group_0_cache = torch.zeros(num_gpu_blocks, 2) + group_1_cache = torch.zeros(num_gpu_blocks, 2) + connector.register_kv_caches( + { + "group_0_layer": group_0_cache, + "group_1_layer": group_1_cache, + } + ) + connector.bind_connector_metadata( + DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([1], [5]), block_size)}) + ) + + connector.start_load_kv( + ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}) + ) + + assert torch.allclose(group_0_cache[1], torch.tensor(0.015)) + assert torch.count_nonzero(group_0_cache[5]) == 0 + assert torch.count_nonzero(group_1_cache[1]) == 0 + assert torch.allclose(group_1_cache[5], torch.tensor(0.015)) + + +def test_decode_bench_connector_uses_per_group_block_sizes(): + """Scheduler selects only the blocks covering each group's token span.""" + vllm_config = create_vllm_config( + block_size=16, + max_num_batched_tokens=1000, + kv_connector="DecodeBenchConnector", + ) + kv_cache_groups = [ + KVCacheGroupSpec( + [f"group_{group_idx}_layer"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ) + for group_idx, block_size in enumerate((16, 32)) + ] + connector = DecodeBenchConnector( + vllm_config, + KVConnectorRole.SCHEDULER, + KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=kv_cache_groups, + ), + ) + request = MagicMock(request_id="request") + blocks = MagicMock() + blocks.get_block_ids.return_value = ([0, 1], [2, 3]) + + connector.update_state_after_alloc(request, blocks, num_external_tokens=17) + metadata = connector.build_connector_meta(MagicMock()) + + assert isinstance(metadata, DecodeBenchConnectorMetadata) + block_ids_per_group, num_tokens = metadata.reqs_to_fill["request"] + assert block_ids_per_group == ([0, 1], [2]) + assert num_tokens == 17 + + def test_decode_bench_connector_no_refill(): """Test that DecodeBenchConnector only fills once per request.""" block_size = 16 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index 4485c2ecd0..999e948eeb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -69,8 +69,8 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata): # request_id -> (block_ids_per_group, num_tokens_to_fill) # block_ids_per_group is a tuple of lists, one per KV cache group - # For standard attention: single group, e.g., ([1, 2, 3],) - # For MLA: multiple groups, e.g., ([1, 2], [1, 2]) + # One group: ([1, 2, 3],) + # Multiple groups: ([1, 2], [5, 6]) reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]] @@ -95,9 +95,13 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA): self.connector_worker: DecodeBenchConnectorWorker | None = None if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler = DecodeBenchConnectorScheduler(vllm_config) + self.connector_scheduler = DecodeBenchConnectorScheduler( + vllm_config, kv_cache_config + ) elif role == KVConnectorRole.WORKER: - self.connector_worker = DecodeBenchConnectorWorker(vllm_config) + self.connector_worker = DecodeBenchConnectorWorker( + vllm_config, kv_cache_config + ) # ============================== # Worker-side methods @@ -182,9 +186,11 @@ class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA): class DecodeBenchConnectorScheduler: """Scheduler-side implementation for DecodeBenchConnector.""" - def __init__(self, vllm_config: "VllmConfig"): + def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): self.vllm_config = vllm_config - self.block_size = vllm_config.cache_config.block_size + self.group_block_sizes = tuple( + group.kv_cache_spec.block_size for group in kv_cache_config.kv_cache_groups + ) # Track which requests have already been filled self._filled_requests: set[str] = set() @@ -236,8 +242,7 @@ class DecodeBenchConnectorScheduler: Called after blocks are allocated. Store the block IDs so we can fill them with dummy values. - Supports both standard attention (single KV cache group) and MLA - (multiple KV cache groups). + Supports both single- and multi-group KV cache configurations. """ req_id = request.request_id @@ -246,18 +251,14 @@ class DecodeBenchConnectorScheduler: # Get the block IDs that were allocated # block_groups is a tuple of lists, one per KV cache group - # For standard attention: 1 group - # For MLA: multiple groups (one per attention type) block_groups = blocks.get_block_ids() - # Calculate how many blocks we need to fill - # num_external_tokens are the tokens we said we'd provide - num_blocks_to_fill = cdiv(num_external_tokens, self.block_size) - - # Extract the first num_blocks_to_fill blocks from each group - # All groups should have the same block IDs for the same request + # Extract the blocks covering num_external_tokens from each group block_ids_per_group = tuple( - group_blocks[:num_blocks_to_fill] for group_blocks in block_groups + group_blocks[: cdiv(num_external_tokens, group_block_size)] + for group_blocks, group_block_size in zip( + block_groups, self.group_block_sizes, strict=True + ) ) # Store the blocks to fill for all group. _pending_fills doesn't need cleanup @@ -269,9 +270,9 @@ class DecodeBenchConnectorScheduler: self._filled_requests.add(req_id) logger.debug( - "DecodeBenchConnector: Allocated %d blocks across %d KV cache groups " + "DecodeBenchConnector: Allocated %s blocks across %d KV cache groups " "for request %s", - num_blocks_to_fill, + tuple(len(block_ids) for block_ids in block_ids_per_group), len(block_groups), req_id, ) @@ -300,7 +301,7 @@ class DecodeBenchConnectorScheduler: class DecodeBenchConnectorWorker: """Worker-side implementation for DecodeBenchConnector.""" - def __init__(self, vllm_config: "VllmConfig"): + def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"): self.vllm_config = vllm_config self.block_size = vllm_config.cache_config.block_size @@ -314,17 +315,15 @@ class DecodeBenchConnectorWorker: self.kv_caches: dict[str, torch.Tensor] | None = None # Mapping from KV cache group index to list of layer names in that group - self.group_to_layers: dict[int, list[str]] | None = None + self.group_to_layers = { + group_idx: list(group.layer_names) + for group_idx, group in enumerate(kv_cache_config.kv_cache_groups) + } def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - """Store references to the KV cache tensors and build group mapping.""" + """Store references to the KV cache tensors.""" self.kv_caches = kv_caches - # For simplicity, assume all layers belong to group 0 (standard attention) - # For MLA models with multiple groups, the metadata will handle the mapping - # We just need to fill the blocks specified in the metadata - self.group_to_layers = {0: list(kv_caches.keys())} - logger.debug( "DecodeBenchConnector: Registered %d KV cache layers", len(kv_caches), @@ -337,13 +336,12 @@ class DecodeBenchConnectorWorker: This simulates having a populated KV cache from a prefill phase, allowing decode performance testing with larger context sizes. - Supports both standard attention (single group) and MLA (multiple groups). + Supports both single- and multi-group KV cache configurations. """ if not metadata.reqs_to_fill: return assert self.kv_caches is not None, "KV caches must be registered before filling" - assert self.group_to_layers is not None, "Group mapping must be initialized" for req_id, (block_ids_per_group, num_tokens) in metadata.reqs_to_fill.items(): # Fill blocks for each KV cache group @@ -372,7 +370,6 @@ class DecodeBenchConnectorWorker: return assert self.kv_caches is not None - assert self.group_to_layers is not None # Get the layers that belong to this group layer_names = self.group_to_layers.get(group_idx, [])