[Bugfix] Restore KV cache metadata GET method for external event consumer (#56925)

Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-09-17 23:01:35 +00:00
committed by GitHub
parent e0050f287a
commit ccde9c61e9
2 changed files with 101 additions and 1 deletions
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.engine.core import EngineCore
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
MLAAttentionSpec,
)
pytestmark = pytest.mark.cpu_test
BASE_BLOCK_SIZE = 1536
DCP_WORLD_SIZE = 8
def _make_engine_core_with_dcp_manager() -> EngineCore:
mla_spec = MLAAttentionSpec(
block_size=BASE_BLOCK_SIZE,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
mamba_spec = MambaSpec(
block_size=BASE_BLOCK_SIZE,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["mla"], mla_spec),
KVCacheGroupSpec(["mamba"], mamba_spec),
],
)
manager = KVCacheManager(
config,
max_model_len=BASE_BLOCK_SIZE * DCP_WORLD_SIZE * 2,
scheduler_block_size=BASE_BLOCK_SIZE * DCP_WORLD_SIZE,
hash_block_size=BASE_BLOCK_SIZE,
dcp_world_size=DCP_WORLD_SIZE,
)
scheduler = SimpleNamespace(kv_cache_config=config, kv_cache_manager=manager)
engine_core = object.__new__(EngineCore)
engine_core.scheduler = scheduler
return engine_core
def test_kv_cache_group_metadata_uses_effective_dcp_block_sizes() -> None:
engine_core = _make_engine_core_with_dcp_manager()
metadata = engine_core.get_kv_cache_group_metadata()
# DCP scales attention blocks but does not scale replicated Mamba state.
assert [item["block_size"] for item in metadata] == [12288, 1536]
assert [item["kind"] for item in metadata] == ["mla_attention", "mamba"]
managers = engine_core.scheduler.kv_cache_manager.coordinator.single_type_managers
assert [item["block_size"] for item in metadata] == [
manager.block_size for manager in managers
]
+31 -1
View File
@@ -85,7 +85,11 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import (
EngineCoreSentinel,
fault_tolerant_wrapper,
)
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
get_kv_cache_spec_kind,
is_full_attention_spec,
)
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
@@ -457,6 +461,32 @@ class EngineCore:
supported_pooling_tasks,
)
def get_kv_cache_group_metadata(self) -> list[dict[str, int | str | None]]:
"""Return serializable KV cache metadata for external event consumers
(e.g., Dynamo). This function has no call sites within vLLM but needs to
be kept.
"""
kv_cache_config = getattr(self.scheduler, "kv_cache_config", None)
if kv_cache_config is None:
return []
kv_cache_manager = cast(Any, self.scheduler).kv_cache_manager
managers = kv_cache_manager.coordinator.single_type_managers
metadata: list[dict[str, int | str | None]] = []
for group_idx, (group, manager) in enumerate(
zip(kv_cache_config.kv_cache_groups, managers, strict=True)
):
spec = group.kv_cache_spec
metadata.append(
{
"group_idx": group_idx,
"kind": get_kv_cache_spec_kind(spec).value,
"block_size": manager.block_size,
"sliding_window": getattr(spec, "sliding_window", None),
}
)
return metadata
def add_request(self, request: Request, request_wave: int = 0):
"""Add request to the scheduler.