[Core][KV Connector] Resolve connector block tables for every scheduled request (#54853)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Zhewen Li
Claude Fable 5.1
Codex
parent
a97dacb710
commit
0b066293f3
@@ -226,7 +226,8 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
req_ids=set(),
|
||||
resolve_block_ids={}.__getitem__,
|
||||
boundary_state_offloads={"req": [(1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
@@ -285,7 +286,8 @@ def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
|
||||
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
req_ids=set(),
|
||||
resolve_block_ids={}.__getitem__,
|
||||
boundary_state_offloads={"req": [(1, 98, 16), (1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
@@ -310,7 +312,8 @@ def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
block_ids={},
|
||||
req_ids=set(),
|
||||
resolve_block_ids={}.__getitem__,
|
||||
boundary_state_offloads={"req": [(1, 99, 16)]},
|
||||
)
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
@@ -4495,7 +4498,8 @@ class TestMambaHybridOffloadServing:
|
||||
num_scheduled_tokens={"A": self.PROMPT_TOKENS},
|
||||
finished_req_ids=set(),
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
req_ids=set(),
|
||||
resolve_block_ids={}.__getitem__,
|
||||
boundary_state_offloads={"A": [(1, 101, self.MAMBA_BLOCK)]},
|
||||
),
|
||||
)
|
||||
|
||||
@@ -57,8 +57,10 @@ def _make_connector_block_state(
|
||||
block_ids: tuple[list[int], ...] | None = None,
|
||||
offloads: list[tuple[int, int, int]] | None = None,
|
||||
) -> KVConnectorBlockState:
|
||||
tables = {} if block_ids is None else {"req-0": block_ids}
|
||||
return KVConnectorBlockState(
|
||||
block_ids={} if block_ids is None else {"req-0": block_ids},
|
||||
req_ids=set(tables),
|
||||
resolve_block_ids=tables.__getitem__,
|
||||
boundary_state_offloads=({} if offloads is None else {"req-0": offloads}),
|
||||
)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
|
||||
import vllm.plugins as plugins_module
|
||||
from tests.v1.core.utils import create_requests, create_scheduler
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import (
|
||||
KVConnectorFactory,
|
||||
)
|
||||
@@ -15,9 +16,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
@@ -54,7 +56,11 @@ class DummyKVConnector(KVConnectorBase_V1):
|
||||
)
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None
|
||||
block_ids_by_req = block_state.block_ids
|
||||
block_ids_by_req = {}
|
||||
for req_id in block_state.req_ids:
|
||||
block_ids = block_state.get_block_ids(req_id)
|
||||
assert block_ids is not None
|
||||
block_ids_by_req[req_id] = block_ids
|
||||
return DummyConnectorMetadata(
|
||||
block_hashes_by_req=block_hashes_by_req,
|
||||
block_ids_by_req=block_ids_by_req,
|
||||
@@ -75,11 +81,12 @@ class DummyKVConnector(KVConnectorBase_V1):
|
||||
|
||||
def _my_plugin():
|
||||
"""Registers the dummy KV connector and overrides _build_kv_connector_meta"""
|
||||
KVConnectorFactory.register_connector(
|
||||
"DummyKVConnector",
|
||||
__name__,
|
||||
DummyKVConnector.__name__,
|
||||
)
|
||||
if "DummyKVConnector" not in KVConnectorFactory._registry:
|
||||
KVConnectorFactory.register_connector(
|
||||
"DummyKVConnector",
|
||||
__name__,
|
||||
DummyKVConnector.__name__,
|
||||
)
|
||||
|
||||
def _custom_build_kv_connector_meta(
|
||||
self, connector: KVConnectorBase_V1, scheduler_output: SchedulerOutput
|
||||
@@ -126,7 +133,8 @@ def test_connector_receives_block_hashes(_load_plugin):
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
# Verify the connector metadata was built with block hashes.
|
||||
# Exercise worker serialization without opening sockets or shared memory.
|
||||
MessageQueue(n_reader=0, n_local_reader=0).enqueue(output)
|
||||
meta = output.kv_connector_metadata
|
||||
assert isinstance(meta, DummyConnectorMetadata)
|
||||
assert len(meta.block_hashes_by_req) == 3
|
||||
@@ -144,3 +152,63 @@ def test_connector_receives_block_hashes(_load_plugin):
|
||||
}
|
||||
assert meta.block_ids_by_req == expected_block_ids
|
||||
assert output.kv_connector_block_state is None
|
||||
|
||||
|
||||
def _make_model_runner_output(scheduler: Scheduler) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in scheduler.running],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(scheduler.running)},
|
||||
sampled_token_ids=[[1000]] * len(scheduler.running),
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
|
||||
def test_connector_block_state_covers_steps_without_new_blocks(_load_plugin):
|
||||
"""A store save lands on the step that fills a block, which allocates none.
|
||||
|
||||
The connector must still be able to look up that request's current table.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
use_kv_connector="DummyKVConnector", block_size=block_size
|
||||
)
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=3 * block_size - 1, block_size=block_size
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_from_output(output, _make_model_runner_output(scheduler))
|
||||
|
||||
# The first decode token fills the third block: no allocation this step.
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_cached_reqs.req_ids == [request.request_id]
|
||||
[new_block_ids] = output.scheduled_cached_reqs.new_block_ids
|
||||
assert not new_block_ids
|
||||
|
||||
meta = output.kv_connector_metadata
|
||||
assert isinstance(meta, DummyConnectorMetadata)
|
||||
block_ids = meta.block_ids_by_req.get(request.request_id)
|
||||
assert block_ids is not None
|
||||
assert block_ids == scheduler.kv_cache_manager.get_block_ids(request.request_id)
|
||||
assert len(block_ids[0]) == 3
|
||||
|
||||
|
||||
def test_connector_block_state_resolves_only_offered_requests():
|
||||
tables = {"a": ([1, 2],), "b": ([3],)}
|
||||
resolve_block_ids = MagicMock(side_effect=tables.__getitem__)
|
||||
block_state = KVConnectorBlockState(
|
||||
req_ids={"a"},
|
||||
resolve_block_ids=resolve_block_ids,
|
||||
boundary_state_offloads={},
|
||||
)
|
||||
|
||||
assert block_state.get_block_ids("b") is None
|
||||
resolve_block_ids.assert_not_called()
|
||||
assert block_state.get_block_ids("a") == ([1, 2],)
|
||||
resolve_block_ids.assert_called_once_with("a")
|
||||
|
||||
tables["a"] = ([4, 5],)
|
||||
assert block_state.get_block_ids("a") == ([4, 5],)
|
||||
|
||||
@@ -434,7 +434,7 @@ class MooncakeStoreScheduler:
|
||||
"Current block tables are required for Mooncake store jobs"
|
||||
)
|
||||
for req_meta in save_metas:
|
||||
block_ids = block_state.block_ids.get(req_meta.req_id)
|
||||
block_ids = block_state.get_block_ids(req_meta.req_id)
|
||||
assert block_ids is not None, (
|
||||
f"Missing current block table for store request {req_meta.req_id}"
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -209,11 +210,18 @@ class ScheduledEncoderInputStats:
|
||||
class KVConnectorBlockState:
|
||||
"""Scheduler-local block state offered to a producer-side KV connector."""
|
||||
|
||||
# Authoritative current block-table snapshots.
|
||||
block_ids: dict[str, tuple[list[int], ...]]
|
||||
# Requests scheduled this step and requests with a boundary-state hand-off.
|
||||
req_ids: set[str]
|
||||
# Resolve on access to avoid copying tables the connector never reads.
|
||||
resolve_block_ids: Callable[[str], tuple[list[int], ...]]
|
||||
# Exact Mamba "align" boundary-state hand-offs.
|
||||
boundary_state_offloads: dict[str, list[tuple[int, int, int]]]
|
||||
|
||||
def get_block_ids(self, req_id: str) -> tuple[list[int], ...] | None:
|
||||
if req_id not in self.req_ids:
|
||||
return None
|
||||
return self.resolve_block_ids(req_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerOutput:
|
||||
|
||||
@@ -1359,24 +1359,16 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
kv_connector_block_state = None
|
||||
if self.connector is not None:
|
||||
snapshot_req_ids = {req.req_id for req in new_reqs_data}
|
||||
snapshot_req_ids.update(
|
||||
req_id
|
||||
for req_id, block_ids in zip(
|
||||
cached_reqs_data.req_ids,
|
||||
cached_reqs_data.new_block_ids,
|
||||
strict=True,
|
||||
)
|
||||
if block_ids
|
||||
)
|
||||
snapshot_req_ids.update(
|
||||
# Any request scheduled this step can become a connector job now,
|
||||
# not only the ones that were allocated blocks: a store save lands
|
||||
# on the step that fills a block, which allocated none.
|
||||
block_state_req_ids = set(num_scheduled_tokens)
|
||||
block_state_req_ids.update(
|
||||
req_id for req_id in boundary_state_offloads if req_id in self.requests
|
||||
)
|
||||
kv_connector_block_state = KVConnectorBlockState(
|
||||
block_ids={
|
||||
req_id: self.kv_cache_manager.get_block_ids(req_id)
|
||||
for req_id in snapshot_req_ids
|
||||
},
|
||||
req_ids=block_state_req_ids,
|
||||
resolve_block_ids=self.kv_cache_manager.get_block_ids,
|
||||
boundary_state_offloads=boundary_state_offloads,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user