diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index d01d0f258a..14320c367c 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -226,7 +226,8 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources(): output = SimpleNamespace( kv_connector_block_state=KVConnectorBlockState( - block_ids={}, + req_ids=set(), + resolve_block_ids={}.__getitem__, boundary_state_offloads={"req": [(1, 99, 28)]}, ) ) @@ -285,7 +286,8 @@ def test_aligned_boundary_store_uses_exact_source_with_partial_tail(): output = SimpleNamespace( kv_connector_block_state=KVConnectorBlockState( - block_ids={}, + req_ids=set(), + resolve_block_ids={}.__getitem__, boundary_state_offloads={"req": [(1, 98, 16), (1, 99, 28)]}, ) ) @@ -310,7 +312,8 @@ def test_aligned_boundary_store_flushes_before_cow_destination_reuse(): output = SchedulerOutput.make_empty() output.kv_connector_block_state = KVConnectorBlockState( - block_ids={}, + req_ids=set(), + resolve_block_ids={}.__getitem__, boundary_state_offloads={"req": [(1, 99, 16)]}, ) meta = scheduler.build_connector_meta(output) @@ -4495,7 +4498,8 @@ class TestMambaHybridOffloadServing: num_scheduled_tokens={"A": self.PROMPT_TOKENS}, finished_req_ids=set(), kv_connector_block_state=KVConnectorBlockState( - block_ids={}, + req_ids=set(), + resolve_block_ids={}.__getitem__, boundary_state_offloads={"A": [(1, 101, self.MAMBA_BLOCK)]}, ), ) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 7990c4997c..67fa0c88aa 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -57,8 +57,10 @@ def _make_connector_block_state( block_ids: tuple[list[int], ...] | None = None, offloads: list[tuple[int, int, int]] | None = None, ) -> KVConnectorBlockState: + tables = {} if block_ids is None else {"req-0": block_ids} return KVConnectorBlockState( - block_ids={} if block_ids is None else {"req-0": block_ids}, + req_ids=set(tables), + resolve_block_ids=tables.__getitem__, boundary_state_offloads=({} if offloads is None else {"req-0": offloads}), ) diff --git a/tests/v1/kv_connector/unit/test_scheduler_kv_connector_override.py b/tests/v1/kv_connector/unit/test_scheduler_kv_connector_override.py index 6b4d2f894d..0ac1831bf0 100644 --- a/tests/v1/kv_connector/unit/test_scheduler_kv_connector_override.py +++ b/tests/v1/kv_connector/unit/test_scheduler_kv_connector_override.py @@ -6,6 +6,7 @@ import pytest import vllm.plugins as plugins_module from tests.v1.core.utils import create_requests, create_scheduler +from vllm.distributed.device_communicators.shm_broadcast import MessageQueue from vllm.distributed.kv_transfer.kv_connector.factory import ( KVConnectorFactory, ) @@ -15,9 +16,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( ) from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.kv_cache_utils import BlockHash -from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request @@ -54,7 +56,11 @@ class DummyKVConnector(KVConnectorBase_V1): ) block_state = scheduler_output.kv_connector_block_state assert block_state is not None - block_ids_by_req = block_state.block_ids + block_ids_by_req = {} + for req_id in block_state.req_ids: + block_ids = block_state.get_block_ids(req_id) + assert block_ids is not None + block_ids_by_req[req_id] = block_ids return DummyConnectorMetadata( block_hashes_by_req=block_hashes_by_req, block_ids_by_req=block_ids_by_req, @@ -75,11 +81,12 @@ class DummyKVConnector(KVConnectorBase_V1): def _my_plugin(): """Registers the dummy KV connector and overrides _build_kv_connector_meta""" - KVConnectorFactory.register_connector( - "DummyKVConnector", - __name__, - DummyKVConnector.__name__, - ) + if "DummyKVConnector" not in KVConnectorFactory._registry: + KVConnectorFactory.register_connector( + "DummyKVConnector", + __name__, + DummyKVConnector.__name__, + ) def _custom_build_kv_connector_meta( self, connector: KVConnectorBase_V1, scheduler_output: SchedulerOutput @@ -126,7 +133,8 @@ def test_connector_receives_block_hashes(_load_plugin): output = scheduler.schedule() - # Verify the connector metadata was built with block hashes. + # Exercise worker serialization without opening sockets or shared memory. + MessageQueue(n_reader=0, n_local_reader=0).enqueue(output) meta = output.kv_connector_metadata assert isinstance(meta, DummyConnectorMetadata) assert len(meta.block_hashes_by_req) == 3 @@ -144,3 +152,63 @@ def test_connector_receives_block_hashes(_load_plugin): } assert meta.block_ids_by_req == expected_block_ids assert output.kv_connector_block_state is None + + +def _make_model_runner_output(scheduler: Scheduler) -> ModelRunnerOutput: + return ModelRunnerOutput( + req_ids=[req.request_id for req in scheduler.running], + req_id_to_index={req.request_id: i for i, req in enumerate(scheduler.running)}, + sampled_token_ids=[[1000]] * len(scheduler.running), + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ) + + +def test_connector_block_state_covers_steps_without_new_blocks(_load_plugin): + """A store save lands on the step that fills a block, which allocates none. + + The connector must still be able to look up that request's current table. + """ + block_size = 16 + scheduler = create_scheduler( + use_kv_connector="DummyKVConnector", block_size=block_size + ) + (request,) = create_requests( + num_requests=1, num_tokens=3 * block_size - 1, block_size=block_size + ) + scheduler.add_request(request) + + output = scheduler.schedule() + scheduler.update_from_output(output, _make_model_runner_output(scheduler)) + + # The first decode token fills the third block: no allocation this step. + output = scheduler.schedule() + assert output.scheduled_cached_reqs.req_ids == [request.request_id] + [new_block_ids] = output.scheduled_cached_reqs.new_block_ids + assert not new_block_ids + + meta = output.kv_connector_metadata + assert isinstance(meta, DummyConnectorMetadata) + block_ids = meta.block_ids_by_req.get(request.request_id) + assert block_ids is not None + assert block_ids == scheduler.kv_cache_manager.get_block_ids(request.request_id) + assert len(block_ids[0]) == 3 + + +def test_connector_block_state_resolves_only_offered_requests(): + tables = {"a": ([1, 2],), "b": ([3],)} + resolve_block_ids = MagicMock(side_effect=tables.__getitem__) + block_state = KVConnectorBlockState( + req_ids={"a"}, + resolve_block_ids=resolve_block_ids, + boundary_state_offloads={}, + ) + + assert block_state.get_block_ids("b") is None + resolve_block_ids.assert_not_called() + assert block_state.get_block_ids("a") == ([1, 2],) + resolve_block_ids.assert_called_once_with("a") + + tables["a"] = ([4, 5],) + assert block_state.get_block_ids("a") == ([4, 5],) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 1c06560d61..ae61710e25 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -434,7 +434,7 @@ class MooncakeStoreScheduler: "Current block tables are required for Mooncake store jobs" ) for req_meta in save_metas: - block_ids = block_state.block_ids.get(req_meta.req_id) + block_ids = block_state.get_block_ids(req_meta.req_id) assert block_ids is not None, ( f"Missing current block table for store request {req_meta.req_id}" ) diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index c9d969cd34..77904f880a 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable from dataclasses import dataclass from functools import cached_property from typing import TYPE_CHECKING @@ -209,11 +210,18 @@ class ScheduledEncoderInputStats: class KVConnectorBlockState: """Scheduler-local block state offered to a producer-side KV connector.""" - # Authoritative current block-table snapshots. - block_ids: dict[str, tuple[list[int], ...]] + # Requests scheduled this step and requests with a boundary-state hand-off. + req_ids: set[str] + # Resolve on access to avoid copying tables the connector never reads. + resolve_block_ids: Callable[[str], tuple[list[int], ...]] # Exact Mamba "align" boundary-state hand-offs. boundary_state_offloads: dict[str, list[tuple[int, int, int]]] + def get_block_ids(self, req_id: str) -> tuple[list[int], ...] | None: + if req_id not in self.req_ids: + return None + return self.resolve_block_ids(req_id) + @dataclass class SchedulerOutput: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 77e495b699..275dc77b6a 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1359,24 +1359,16 @@ class Scheduler(SchedulerInterface): kv_connector_block_state = None if self.connector is not None: - snapshot_req_ids = {req.req_id for req in new_reqs_data} - snapshot_req_ids.update( - req_id - for req_id, block_ids in zip( - cached_reqs_data.req_ids, - cached_reqs_data.new_block_ids, - strict=True, - ) - if block_ids - ) - snapshot_req_ids.update( + # Any request scheduled this step can become a connector job now, + # not only the ones that were allocated blocks: a store save lands + # on the step that fills a block, which allocated none. + block_state_req_ids = set(num_scheduled_tokens) + block_state_req_ids.update( req_id for req_id in boundary_state_offloads if req_id in self.requests ) kv_connector_block_state = KVConnectorBlockState( - block_ids={ - req_id: self.kv_cache_manager.get_block_ids(req_id) - for req_id in snapshot_req_ids - }, + req_ids=block_state_req_ids, + resolve_block_ids=self.kv_cache_manager.get_block_ids, boundary_state_offloads=boundary_state_offloads, )