From f007cceb422599330495b202632d78c7b3b89e61 Mon Sep 17 00:00:00 2001 From: Chang Guo Date: Mon, 20 Jul 2026 06:41:58 -0700 Subject: [PATCH] [KV Offload] Support self-describing KV events with TieringOffloadingSpec (#48679) Signed-off-by: Change72 Co-authored-by: OpenAI Codex --- docs/features/kv_offloading_usage.md | 2 +- .../unit/offloading_connector/test_events.py | 157 +++++++++- .../offloading_connector/test_scheduler.py | 286 ++++++++++++++++-- .../kv_connector/v1/offloading/events.py | 40 ++- .../kv_connector/v1/offloading/scheduler.py | 24 +- vllm/v1/kv_offload/tiering/spec.py | 8 - 6 files changed, 452 insertions(+), 65 deletions(-) diff --git a/docs/features/kv_offloading_usage.md b/docs/features/kv_offloading_usage.md index 0ae7c23f40..b5c9d68e26 100644 --- a/docs/features/kv_offloading_usage.md +++ b/docs/features/kv_offloading_usage.md @@ -75,7 +75,7 @@ vllm serve \ | `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. | | `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). | | `offload_prompt_only` | no | `true` | both | If `true`, only prompt (prefill) blocks are offloaded; decode blocks are skipped. | -| `self_describing_kv_events` | no | `false` | single-tier | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. Currently rejected by `TieringOffloadingSpec`. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. | +| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. | | `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). | ## Secondary Tiers diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index a23fb95b56..5c172454b5 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -7,7 +7,13 @@ import torch from tests.v1.kv_connector.unit.utils import create_vllm_config from vllm.config import KVEventsConfig, KVTransferConfig -from vllm.distributed.kv_events import MEDIUM_CPU, MEDIUM_FS, BlockRemoved, BlockStored +from vllm.distributed.kv_events import ( + MEDIUM_CPU, + MEDIUM_FS, + MEDIUM_OBJ, + BlockRemoved, + BlockStored, +) from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import ( build_offloading_config, ) @@ -62,8 +68,9 @@ def _wire_hash(block_hash: BlockHash): return maybe_convert_block_hash(block_hash) -def _request(*, block_hashes: list[BlockHash], token_count: int): +def _request(*, block_hashes: list[BlockHash], token_count: int, req_id: str = "req"): req = MagicMock() + req.request_id = req_id req.block_hashes = block_hashes req.all_token_ids = list(range(1, token_count + 1)) req.lora_request = None @@ -104,10 +111,32 @@ def _record_chunks( return keys +def _record_lookup_chunks( + tracker: OffloadingEventsTracker, + req, + group_config: GroupOffloadConfig, + num_chunks: int, +) -> list[OffloadKey]: + keys: list[OffloadKey] = [] + hbf = group_config.hashes_per_chunk + for chunk_idx in range(num_chunks): + tail_hash = req.block_hashes[(chunk_idx + 1) * hbf - 1] + assert tail_hash is not None + key = make_offload_key(tail_hash, group_config.group_idx) + tracker.record_lookup( + req, + group_config, + chunk_idx, + key, + ) + keys.append(key) + return keys + + def _stored_event( keys: list[OffloadKey], - locality: Locality | None = None, medium: str = _CPU_MEDIUM, + locality: Locality | None = None, ) -> OffloadingEvent: return OffloadingEvent( keys=keys, @@ -119,8 +148,8 @@ def _stored_event( def _removed_event( keys: list[OffloadKey], - locality: Locality | None = None, medium: str = _CPU_MEDIUM, + locality: Locality | None = None, ) -> OffloadingEvent: return OffloadingEvent( keys=keys, @@ -130,6 +159,21 @@ def _removed_event( ) +def _lookup_chunk() -> tuple[ + OffloadingEventsTracker, MagicMock, GroupOffloadConfig, OffloadKey +]: + tracker = _tracker() + req = _request(block_hashes=[_hash(0)], token_count=4) + group_config = _group_config() + key = _record_lookup_chunks( + tracker, + req, + group_config, + num_chunks=1, + )[0] + return tracker, req, group_config, key + + def test_take_events_forwards_locality_to_rich_store(): tracker = _tracker() req = _request(block_hashes=[_hash(0)], token_count=4) @@ -220,18 +264,37 @@ def test_take_events_publishes_routable_block_stored(): assert len(tracker._pending_event_metadata) == 6 -def test_take_events_factor_gt_1_chunk_store_and_remove(): +def test_promotion_emits_full_cpu_stored_event(): + tracker, _, _, key = _lookup_chunk() + + [event] = tracker.take_events([_stored_event([key])]) + + assert isinstance(event, BlockStored) + assert event.medium == MEDIUM_CPU + assert event.block_hashes == [_wire_hash(_hash(0))] + assert event.parent_block_hash is None + assert event.token_ids == [1, 2, 3, 4] + assert event.block_size == 4 + assert event.lora_id is None + assert event.lora_name is None + assert event.extra_keys is None + assert event.group_idx == 0 + assert event.kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value + assert event.kv_cache_spec_sliding_window is None + + +def test_lookup_promotion_factor_gt_1_store_and_remove(): block_size = 4 - blocks_per_chunk = 3 + blocks_per_chunk = 2 tracker = _tracker() group_config = _group_config( block_size=block_size, blocks_per_chunk=blocks_per_chunk ) req = _request( - block_hashes=[_hash(i) for i in range(6)], + block_hashes=[_hash(i) for i in range(4)], token_count=block_size * blocks_per_chunk * 2, ) - keys = _record_chunks(tracker, req, group_config, num_chunks=2) + keys = _record_lookup_chunks(tracker, req, group_config, num_chunks=2) stored = list(tracker.take_events([_stored_event(keys)])) assert len(stored) == 2 @@ -293,6 +356,7 @@ def test_take_events_opt_out_keeps_placeholders(): group_config = _group_config() req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12) keys = _record_chunks(tracker, req, group_config, num_chunks=3) + _record_lookup_chunks(tracker, req, group_config, num_chunks=3) assert not tracker.self_describing_enabled assert not tracker._pending_event_metadata @@ -315,11 +379,21 @@ def test_take_events_opt_out_keeps_placeholders(): assert len(events[3].block_hashes) == 3 -def test_record_store_skips_sliding_window_group(): +@pytest.mark.parametrize( + "sliding_window_size_in_chunks", + [1, 2], + ids=["ssm", "sliding-window"], +) +def test_event_metadata_skips_non_full_attention_group( + sliding_window_size_in_chunks: int, +): tracker = _tracker() - group_config = _group_config(sliding_window_size_in_chunks=2) + group_config = _group_config( + sliding_window_size_in_chunks=sliding_window_size_in_chunks + ) req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12) keys = _record_chunks(tracker, req, group_config, num_chunks=3) + _record_lookup_chunks(tracker, req, group_config, num_chunks=3) assert not tracker._pending_event_metadata @@ -329,6 +403,57 @@ def test_record_store_skips_sliding_window_group(): assert events[0].block_size == 0 +def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit(): + tracker = _tracker() + block_hashes = [_hash(0), _hash(1)] + req = _request(block_hashes=block_hashes, token_count=8) + group_config = _group_config(blocks_per_chunk=2) + key = _record_chunks(tracker, req, group_config, num_chunks=1)[0] + confirmed_meta = tracker._pending_event_metadata[key] + lookup_req = _request( + block_hashes=block_hashes, + token_count=8, + req_id="new-request", + ) + + tracker.record_lookup( + lookup_req, + group_config, + 0, + key, + ) + assert tracker._pending_event_metadata[key] is confirmed_meta + + removed = list(tracker.take_events([_removed_event([key])])) + assert len(removed) == 1 + assert removed[0].block_hashes == [ + _wire_hash(_hash(0)), + _wire_hash(_hash(1)), + ] + + stored = list(tracker.take_events([_stored_event([key])])) + assert len(stored) == 1 + assert stored[0].block_size == 0 + assert stored[0].token_ids == [] + + tracker.record_lookup(lookup_req, group_config, 0, key) + removed = list(tracker.take_events([_removed_event([key])])) + assert removed[0].block_hashes == [ + _wire_hash(_hash(0)), + _wire_hash(_hash(1)), + ] + + +@pytest.mark.parametrize("medium", [MEDIUM_FS, MEDIUM_OBJ]) +def test_secondary_stored_event_does_not_mutate_cpu_metadata(medium: str): + tracker, _, _, key = _lookup_chunk() + expected_metadata = dict(tracker._pending_event_metadata) + + stored = list(tracker.take_events([_stored_event([key], medium)])) + assert stored[0].token_ids == [1, 2, 3, 4] + assert tracker._pending_event_metadata == expected_metadata + + def test_take_events_groups_removed_hashes_by_kv_group(): tracker = _tracker() group0_config = _group_config(group_idx=0, blocks_per_chunk=2) @@ -378,7 +503,7 @@ def test_reset_cache_clears_side_table(): tracker = _tracker() group_config = _group_config() req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12) - _record_chunks(tracker, req, group_config, num_chunks=3) + _record_lookup_chunks(tracker, req, group_config, num_chunks=3) assert tracker._pending_event_metadata @@ -387,7 +512,7 @@ def test_reset_cache_clears_side_table(): assert not tracker._pending_event_metadata -def test_tiering_rejects_self_describing_kv_events(): +def test_tiering_accepts_self_describing_kv_events(): vllm_config = create_vllm_config( block_size=4, max_num_batched_tokens=16, @@ -423,5 +548,9 @@ def test_tiering_rejects_self_describing_kv_events(): ], ) - with pytest.raises(ValueError, match="TieringOffloadingSpec"): - TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + spec = TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + tracker = OffloadingEventsTracker(spec.kv_events_config) + + assert spec.kv_events_config.enable_kv_cache_events + assert spec.kv_events_config.self_describing_kv_events + assert tracker.self_describing_enabled diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index bc5190db00..9c831a2354 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call import pytest import torch @@ -11,6 +11,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import ( to_keys, ) from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID +from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, _ConnectorMetricName, @@ -19,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( OffloadingConnectorScheduler, RequestOffloadState, ) +from vllm.v1.core.kv_cache_utils import BlockHash from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheGroupSpec, @@ -26,6 +28,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.kv_offload.base import ( LookupResult, + OffloadingEvent, OffloadingManager, OffloadPolicy, ReqContext, @@ -143,6 +146,159 @@ def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner): assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0 +def test_max_offload_tokens_zero_does_not_record_pending_lookups(request_runner): + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + ) + runner.manager.lookup.return_value = LookupResult.RETRY + runner.manager.take_events.return_value = [] + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + + runner.new_request( + token_ids=[1] * 12, + kv_transfer_params={"max_offload_tokens": 0}, + ) + runner.run(decoded_tokens=[]) + + tracker = runner.connector_scheduler._events_tracker + assert runner.manager.lookup.call_count == 3 + assert not tracker._pending_event_metadata + assert list(runner.connector_scheduler.take_events()) == [] + + runner.manager.lookup.return_value = LookupResult.MISS + runner.run(decoded_tokens=[EOS_TOKEN_ID]) + + assert not tracker._pending_event_metadata + assert list(runner.connector_scheduler.take_events()) == [] + + +def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal( + request_runner, +): + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + blocks_per_chunk=2, + ) + raw_events: list[OffloadingEvent] = [] + + def take_raw_events(): + yield from raw_events + raw_events.clear() + + runner.manager.lookup.return_value = LookupResult.RETRY + runner.manager.take_events.side_effect = take_raw_events + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output([]) + ) + + runner.new_request(token_ids=[1] * 8) + runner.run(decoded_tokens=[]) + + tracker = runner.connector_scheduler._events_tracker + assert not tracker._pending_event_metadata + key = runner.manager.lookup.call_args.args[0] + req_id = str(runner.req_id) + req_status = runner.connector_scheduler._req_status[req_id] + + runner.scheduler.finish_requests((req_id,), RequestStatus.FINISHED_ABORTED) + + assert not tracker._pending_event_metadata + + raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False)) + events = list(runner.connector_scheduler.take_events()) + assert len(events) == 1 + assert isinstance(events[0], BlockStored) + assert events[0].block_size == 0 + assert events[0].token_ids == [] + + runner.manager.lookup.return_value = LookupResult.HIT + group_config = runner.connector_scheduler.config.kv_group_configs[0] + assert ( + runner.connector_scheduler._maximal_prefix_lookup( + [key], + req_status.req_context, + req_status.req, + group_config, + 0, + ) + == 1 + ) + assert key in tracker._pending_event_metadata + + raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=True)) + [event] = runner.connector_scheduler.take_events() + assert isinstance(event, BlockRemoved) + assert event.medium == MEDIUM_CPU + assert len(event.block_hashes) == 2 + assert key not in tracker._pending_event_metadata + + +@pytest.mark.parametrize("blocks_per_chunk", [1, 2]) +def test_promotion_hit_precedes_stored_event_translation( + request_runner, + blocks_per_chunk: int, +): + runner = request_runner( + block_size=4, + num_gpu_blocks=10, + async_scheduling=False, + blocks_per_chunk=blocks_per_chunk, + ) + token_ids = [1] * 4 * blocks_per_chunk + + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.new_request(token_ids=token_ids) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=tuple(range(blocks_per_chunk)), + ) + runner.scheduler.reset_prefix_cache() + runner.connector_scheduler._events_tracker.reset() + + raw_events: list[OffloadingEvent] = [] + + def lookup(key, req_context): + raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False)) + return LookupResult.HIT + + def take_raw_events(): + yield from raw_events + raw_events.clear() + + runner.manager.lookup.side_effect = lookup + runner.manager.take_events.side_effect = take_raw_events + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output([]) + ) + publisher = MagicMock() + runner.scheduler.kv_event_publisher = publisher + + runner.new_request(token_ids=token_ids) + runner.run( + decoded_tokens=[], + expected_loaded=tuple(range(blocks_per_chunk)), + ) + + events = [ + event + for publish_call in publisher.publish.call_args_list + for event in publish_call.args[0].events + if isinstance(event, BlockStored) and event.medium == MEDIUM_CPU + ] + assert len(events) == 1 + assert len(events[0].block_hashes) == blocks_per_chunk + assert events[0].block_size == 4 + assert events[0].token_ids == token_ids + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_offloading_connector(request_runner, async_scheduling: bool): block_size = 4 @@ -241,7 +397,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool): runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output([]) ) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(0, 1, 2)) # single block lookup with a hit in a middle block @@ -249,7 +405,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool): runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output([]) ) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(3, 4, 5)) @@ -307,7 +463,7 @@ def test_request_preemption(request_runner, async_scheduling: bool): # request should now return from preemption # re-load [0, ..., 8] from the CPU and store [9, 10, 11] - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 3 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 3 runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) @@ -427,7 +583,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: # start a request to load the first block, but don't complete runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * tokens_per_chunk) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run( decoded_tokens=[], complete_transfers=False, @@ -439,7 +595,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: # start a new request to load the same first block runner.new_request(token_ids=[0] * tokens_per_chunk) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run( decoded_tokens=[], complete_transfers=False, @@ -491,7 +647,7 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool): # start a request to load the first block, but don't complete runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * tokens_per_chunk) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run( decoded_tokens=[], complete_transfers=False, @@ -793,73 +949,144 @@ def _make_scheduler_with_lookup( scheduler = object.__new__(OffloadingConnectorScheduler) scheduler.manager = manager + scheduler._events_tracker = MagicMock() return scheduler _EMPTY_REQ_CTX = ReqContext(req_id="") +_LOOKUP_REQ = MagicMock() +_LOOKUP_REQ.request_id = "req" +_LOOKUP_GROUP_CONFIG = MagicMock() + + +def _maximal_lookup(sched, keys, start_chunk_idx: int = 0): + return sched._maximal_prefix_lookup( + keys, + _EMPTY_REQ_CTX, + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + start_chunk_idx, + ) class TestMaximalPrefixLookup: def test_all_hit(self): sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT}) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 2 + assert _maximal_lookup(sched, to_keys([1, 2])) == 2 + + def test_records_absolute_chunk_indices(self): + keys = to_keys([1, 2]) + sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT}) + + assert _maximal_lookup(sched, keys, start_chunk_idx=3) == 2 + assert sched._events_tracker.record_lookup.call_args_list == [ + call( + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + 3, + keys[0], + ), + call( + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + 4, + keys[1], + ), + ] def test_all_miss(self): sched = _make_scheduler_with_lookup({}) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 0 + assert _maximal_lookup(sched, to_keys([1, 2])) == 0 + sched._events_tracker.record_lookup.assert_not_called() def test_partial_prefix(self): sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT}) - assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) == 2 + assert _maximal_lookup(sched, to_keys([1, 2, 3])) == 2 def test_miss_then_hit(self): sched = _make_scheduler_with_lookup({2: LookupResult.HIT}) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 0 + assert _maximal_lookup(sched, to_keys([1, 2])) == 0 def test_single_hit(self): sched = _make_scheduler_with_lookup({1: LookupResult.HIT}) - assert sched._maximal_prefix_lookup(to_keys([1]), _EMPTY_REQ_CTX) == 1 + assert _maximal_lookup(sched, to_keys([1])) == 1 def test_empty(self): sched = _make_scheduler_with_lookup({}) - assert sched._maximal_prefix_lookup([], _EMPTY_REQ_CTX) == 0 + assert _maximal_lookup(sched, []) == 0 + + @pytest.mark.parametrize( + "pending_result", + [LookupResult.RETRY, LookupResult.HIT_PENDING], + ) + def test_pending_result_is_not_recorded( + self, + pending_result: LookupResult, + ): + sched = _make_scheduler_with_lookup({1: pending_result}) + + assert _maximal_lookup(sched, to_keys([1])) is None + sched._events_tracker.record_lookup.assert_not_called() def test_retry_defers(self): + keys = to_keys([1, 2]) sched = _make_scheduler_with_lookup( {1: LookupResult.RETRY, 2: LookupResult.HIT} ) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None + assert _maximal_lookup(sched, keys) is None assert sched.manager.lookup.call_count == 2 + sched._events_tracker.record_lookup.assert_called_once_with( + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + 1, + keys[1], + ) def test_retry_after_hit_defers(self): + keys = to_keys([1, 2]) sched = _make_scheduler_with_lookup( {1: LookupResult.HIT, 2: LookupResult.RETRY} ) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None + assert _maximal_lookup(sched, keys) is None + sched._events_tracker.record_lookup.assert_called_once_with( + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + 0, + keys[0], + ) def test_hit_pending_defers(self): + keys = to_keys([1, 2]) sched = _make_scheduler_with_lookup( {1: LookupResult.HIT_PENDING, 2: LookupResult.HIT} ) - assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None + assert _maximal_lookup(sched, keys) is None assert sched.manager.lookup.call_count == 2 + sched._events_tracker.record_lookup.assert_called_once_with( + _LOOKUP_REQ, + _LOOKUP_GROUP_CONFIG, + 1, + keys[1], + ) def test_hit_pending_does_not_stop_scan(self): """HIT_PENDING defers but does not break — scan continues until miss.""" sched = _make_scheduler_with_lookup( {1: LookupResult.HIT_PENDING, 2: LookupResult.MISS, 3: LookupResult.HIT} ) - assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) is None + assert _maximal_lookup(sched, to_keys([1, 2, 3])) is None assert sched.manager.lookup.call_count == 2 + sched._events_tracker.record_lookup.assert_not_called() def test_retry_stops_at_miss(self): """RETRY is treated as hit for iteration, but miss stops the scan.""" sched = _make_scheduler_with_lookup( {1: LookupResult.RETRY, 2: LookupResult.MISS, 3: LookupResult.HIT} ) - assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) is None + assert _maximal_lookup(sched, to_keys([1, 2, 3])) is None # lookup should have been called for blocks 1 and 2 (stops at miss) assert sched.manager.lookup.call_count == 2 + sched._events_tracker.record_lookup.assert_not_called() class TestSlidingWindowLookup: @@ -1011,7 +1238,7 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling # New request with 2 offloaded chunks; first matches what's in CPU. runner.new_request(token_ids=[0] * tokens_per_chunk * 2) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) @@ -1042,7 +1269,7 @@ def test_loads_do_not_populate_fence_index(request_runner): async_scheduling=False, ) runner.new_request(token_ids=[0] * 12) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.run(decoded_tokens=[], complete_transfers=False) assert runner.connector_scheduler._block_id_to_pending_jobs == {} @@ -1088,7 +1315,7 @@ def test_fence_at_update_state_after_alloc(request_runner): runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * 4) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output([]) ) @@ -1139,7 +1366,7 @@ def test_fence_at_build_store_jobs(request_runner): runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[1] * 4) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 0 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 0 runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output([]) ) @@ -1365,7 +1592,7 @@ def test_reset_cache(request_runner, async_scheduling: bool): # Leave the load in-flight so that reset_cache must flush it. runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * tokens_per_chunk) - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1 runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output([]) ) @@ -1553,9 +1780,7 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner # preemption batch's ModelRunnerOutput is consumed by update_from_output(). free_block_queue.num_free_blocks = num_free_blocks_empty assert runner.scheduler.reset_prefix_cache() - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: len( - key - ) + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: len(keys) readmit_output = runner.scheduler.schedule() @@ -1669,7 +1894,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): runner.scheduler.reset_prefix_cache() runner.new_request(token_ids=[0] * num_tokens + [1]) runner.manager.lookup.return_value = LookupResult.HIT - runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 2 + runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2 runner.run( decoded_tokens=[EOS_TOKEN_ID], # Group 0: full prefix lookup hits 2 offloaded chunks @@ -1840,6 +2065,13 @@ class TestEagle: req.request_id = "test-req" req.num_tokens = num_tokens req.kv_transfer_params = None + num_hash_blocks = max( + len(hashes) * scheduler.config.kv_group_configs[idx].hashes_per_chunk + for idx, hashes in enumerate(offload_keys_per_group) + ) + req.block_hashes = [BlockHash(str(i).encode()) for i in range(num_hash_blocks)] + req.all_token_ids = list(range(num_tokens)) + req.lora_request = None state = RequestOffloadState( config=scheduler.config, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 4543cd6dd3..3c5a9956ca 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -61,9 +61,9 @@ def get_offloading_event_group_spec( @dataclass(slots=True) class _OffloadEventMetadata: - """BlockStored payload snapshot for one OffloadKey, captured at store - time and kept until the matching eviction event. ``medium`` is forwarded - from the OffloadingEvent.""" + """BlockStored payload snapshot for one OffloadKey, captured while the + Request is available and kept until the matching eviction event. ``medium`` + is forwarded from the OffloadingEvent.""" # The chunk's constituent block hashes; the last one is the OffloadKey. block_hashes: tuple[BlockHash, ...] @@ -81,10 +81,11 @@ class _OffloadEventMetadata: class OffloadingEventsTracker: """Tracks offloaded chunks' KV event payloads from store to eviction. - The scheduler calls :meth:`record_store` from ``_build_store_jobs`` - while the ``Request`` is available, and routes the manager's raw - :class:`OffloadingEvent` stream through :meth:`take_events`. All state - is bounded by the CPU pool capacity and cleared by :meth:`reset`. + The scheduler calls :meth:`record_store` from ``_build_store_jobs`` and + :meth:`record_lookup` for ready primary-tier hits while the ``Request`` is + available. Deferred and missing lookups add no state. Under the connector's + supported success-only transfer model, entries follow primary allocations + until CPU removal translation or :meth:`reset`. """ def __init__(self, config: OffloadingKVEventsConfig): @@ -93,8 +94,7 @@ class OffloadingEventsTracker: config.enable_kv_cache_events and config.self_describing_kv_events ) - # OffloadKey -> payload snapshot, kept until the eviction event so - # BlockRemoved can fan out. Bounded: one entry per offloaded chunk. + # OffloadKey -> payload snapshot, kept until CPU removal or reset. self._pending_event_metadata: dict[OffloadKey, _OffloadEventMetadata] = {} def record_store( @@ -116,6 +116,23 @@ class OffloadingEventsTracker: meta = self._build_event_metadata(req, group_config, chunk_idx) self._pending_event_metadata[offload_key] = meta + def record_lookup( + self, + req: Request, + group_config: "GroupOffloadConfig", + chunk_idx: int, + offload_key: OffloadKey, + ) -> None: + """Snapshot metadata for a ready primary-tier lookup hit.""" + if not self.self_describing_enabled: + return + if group_config.sliding_window_size_in_chunks is not None: + return + if offload_key not in self._pending_event_metadata: + self._pending_event_metadata[offload_key] = self._build_event_metadata( + req, group_config, chunk_idx + ) + def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]: """Translate raw OffloadingEvents into self-describing KV events. @@ -165,7 +182,7 @@ class OffloadingEventsTracker: assert len(chunk_hashes) == hbf if group_config.sliding_window_size_in_chunks is not None: - # record_store filters these out before calling this helper. + # The recording methods filter these out before calling this helper. raise AssertionError("self-describing events only support full attention") parent_block_hash: BlockHash | None @@ -232,7 +249,8 @@ class OffloadingEventsTracker: "OffloadingEventsTracker: no event metadata for " "offload key during BlockStored emission; emitting a " "placeholder payload. Expected for non-full-attention " - "groups; otherwise indicates a missing populate path." + "groups and promotions not observed as a primary-tier " + "hit before translation." ) yield self._placeholder_stored(key, event.medium, locality) continue diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 5e98c1266e..aae60ebc91 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -463,15 +463,27 @@ class OffloadingConnectorScheduler: del self._req_status[req_id] def _maximal_prefix_lookup( - self, keys: Iterable[OffloadKey], req_context: ReqContext + self, + keys: Iterable[OffloadKey], + req_context: ReqContext, + req: Request, + group_config: GroupOffloadConfig, + start_chunk_idx: int, ) -> int | None: """Return the number of consecutive offloaded chunks from the start, or None if the backend deferred a lookup.""" hit_count = 0 defer_lookup = False - for key in keys: - match self.manager.lookup(key, req_context): + for local_idx, key in enumerate(keys): + result = self.manager.lookup(key, req_context) + match result: case LookupResult.HIT: + self._events_tracker.record_lookup( + req, + group_config, + start_chunk_idx + local_idx, + key, + ) hit_count += 1 case LookupResult.HIT_PENDING: defer_lookup = True @@ -616,7 +628,11 @@ class OffloadingConnectorScheduler: num_hit_chunks: int | None if sliding_window_size_in_chunks is None: num_hit_chunks = self._maximal_prefix_lookup( - offload_keys, req_status.req_context + offload_keys, + req_status.req_context, + req_status.req, + group_config, + start_chunk_idx, ) else: required_window = sliding_window_size_in_chunks diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 964699968c..7aae3a2eb7 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -136,14 +136,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec): super().__init__(config) # Redeclare for mypy: parent sets this but `--follow-imports skip` hides it self._manager: OffloadingManager | None = None - if self.kv_events_config.self_describing_kv_events: - raise ValueError( - "self_describing_kv_events is not supported by " - "TieringOffloadingSpec. Tier promotions can emit primary-tier " - "store events that do not correspond to GPU store jobs, so the " - "current self-describing side table cannot describe them " - "correctly." - ) # Parse secondary tier configurations self.secondary_tier_configs = self.extra_config.get("secondary_tiers", [])