[Bugfix][KV Connector] Safely fill circular buffers in DecodeBench (#54879)
Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
edc0fb7e03
commit
31e9c13685
@@ -31,6 +31,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -243,6 +244,50 @@ def test_decode_bench_connector_fills_each_hma_group():
|
||||
assert torch.allclose(group_1_cache[5], torch.tensor(0.015))
|
||||
|
||||
|
||||
def test_decode_bench_connector_zero_fills_circular_buffer_groups():
|
||||
"""Circular buffers are zeroed to preserve packed non-floating metadata."""
|
||||
num_gpu_blocks = 8
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=16,
|
||||
max_num_batched_tokens=1000,
|
||||
kv_connector="DecodeBenchConnector",
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["circular_layer"],
|
||||
CircularBufferSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
connector = DecodeBenchConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
kv_cache_config,
|
||||
)
|
||||
circular_cache = torch.ones(num_gpu_blocks, 16, dtype=torch.bfloat16)
|
||||
connector.register_kv_caches({"circular_layer": circular_cache})
|
||||
connector.bind_connector_metadata(
|
||||
DecodeBenchConnectorMetadata(reqs_to_fill={"request": (([5],), 1)})
|
||||
)
|
||||
|
||||
connector.start_load_kv(
|
||||
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
)
|
||||
|
||||
expected_circular = torch.ones_like(circular_cache)
|
||||
expected_circular[5] = 0
|
||||
assert torch.equal(circular_cache, expected_circular)
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_block_sizes():
|
||||
"""Scheduler selects only the blocks covering each group's token span."""
|
||||
vllm_config = create_vllm_config(
|
||||
|
||||
Reference in New Issue
Block a user