From d8de4ae322b54e518b2de771fe28673c6e9664d9 Mon Sep 17 00:00:00 2001 From: liranschour Date: Mon, 31 Aug 2026 13:07:20 +0300 Subject: [PATCH] [Bugfix][KVOffload] P2P tier declares REQUEST_LEVEL on the producer leg (#52912) Signed-off-by: Liran Schour --- .../offloading_connector/test_scheduler.py | 239 +++++++++++++++++- .../unit/offloading_connector/utils.py | 2 +- .../v1/kv_offload/tiering/p2p/test_manager.py | 29 ++- .../tiering/test_tiering_offloading.py | 13 +- vllm/v1/kv_offload/tiering/p2p/manager.py | 8 + 5 files changed, 282 insertions(+), 9 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index e69f71314b..53a3524c25 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -1293,12 +1293,21 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool def _make_scheduler_with_lookup( lookup_results: dict[int, LookupResult], + default: LookupResult = LookupResult.MISS, ) -> OffloadingConnectorScheduler: - """Create an OffloadingConnectorScheduler with a mocked manager.lookup.""" + """Create an OffloadingConnectorScheduler with a mocked manager.lookup. + + Keys are addressed by their integer hash; `default` answers anything else. + """ manager = MagicMock(spec=OffloadingManager) - manager.lookup.side_effect = lambda key, req_context: lookup_results.get( - int(get_offload_block_hash(key).decode()), LookupResult.MISS - ) + + def lookup(key, req_context): + block_hash = get_offload_block_hash(key) + if not block_hash.isdigit(): + return default + return lookup_results.get(int(block_hash.decode()), default) + + manager.lookup.side_effect = lookup scheduler = object.__new__(OffloadingConnectorScheduler) scheduler.manager = manager @@ -1322,6 +1331,32 @@ def _maximal_lookup(sched, keys, start_chunk_idx: int = 0): ) +# Lookups issued, and end index returned, by a window-1 scan over 3 keys all +# resolving to the same result. A result that keeps the streak alive ends the +# scan at the first key; one that resets it makes the scan walk every key, +# which is what widens the demanded set. A new member has to be added here. +_SCAN_BEHAVIOR = { + LookupResult.HIT: (1, 3), + LookupResult.HIT_PENDING: (1, None), + LookupResult.RETRY: (3, None), + LookupResult.MISS: (3, 0), +} + + +@pytest.mark.parametrize("result", list(LookupResult)) +def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult): + """Whether a result keeps a sliding-window streak alive decides how wide + the demanded chunk set gets, so every member needs deliberate behavior.""" + assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior" + expected_lookups, expected_end = _SCAN_BEHAVIOR[result] + + keys = to_keys([1, 2, 3]) + sched = _make_scheduler_with_lookup(dict.fromkeys([1, 2, 3], result)) + + assert sched._sliding_window_lookup(keys, 1, _EMPTY_REQ_CTX) == expected_end + assert len(sched.manager.lookup.call_args_list) == expected_lookups + + class TestMaximalPrefixLookup: def test_all_hit(self): sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT}) @@ -1556,6 +1591,62 @@ class TestSlidingWindowLookup: ) +# --------------------------------------------------------------------------- +# Tests for SWA store pruning vs. load demand +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64]) +@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3]) +@pytest.mark.parametrize("is_eagle_group", [False, True]) +@pytest.mark.parametrize("tail_chunks", [0, 1, 5]) +def test_sliding_window_demand_is_store_reachable( + alignment_chunk_count: int, + sliding_window_chunks: int, + is_eagle_group: bool, + tail_chunks: int, +): + """Every chunk the load path looks up must be one the store path keeps. + + `is_store_reachable_swa_chunk` prunes chunks `_sliding_window_lookup` can + never query, but the two mirror each other by hand: this pins the relation, + where `test_is_store_reachable_swa_chunk` pins only the predicate's values. + A looked-up key is a demanded key, since a secondary-tier hit queues a + promotion immediately. + """ + storable_chunk_count = alignment_chunk_count + tail_chunks + # Both sides widen by one for an unverified EAGLE group. + required_window = sliding_window_chunks + int(is_eagle_group) + + sched = _make_scheduler_with_lookup( + {i: LookupResult.HIT_PENDING for i in range(storable_chunk_count)} + ) + sched._sliding_window_lookup( + to_keys(range(storable_chunk_count)), required_window, _EMPTY_REQ_CTX + ) + + demanded = { + int(get_offload_block_hash(lookup_call.args[0]).decode()) + for lookup_call in sched.manager.lookup.call_args_list + } + assert demanded, "the scan must query something" + unsuppliable = sorted( + chunk_idx + for chunk_idx in demanded + if not is_store_reachable_swa_chunk( + chunk_idx, + storable_chunk_count, + alignment_chunk_count, + sliding_window_chunks, + is_eagle_group, + ) + ) + assert not unsuppliable, ( + f"chunks {unsuppliable} are demanded by the load path but pruned by the " + f"store path, so a warm producer can never supply them" + ) + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool): """With REQUEST_LEVEL policy, all blocks are stored — including prefix hits.""" @@ -2308,6 +2399,146 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool): ) +def _full_and_swa_groups( + full_attn_block_size: int, swa_block_size: int, sliding_window: int +) -> list[KVCacheGroupSpec]: + """Group 0 full attention (MLA-like), group 1 sliding window.""" + return [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=full_attn_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + SlidingWindowSpec( + block_size=swa_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sliding_window, + ), + ), + ] + + +def _demanded_keys(runner, offload_keys_per_group: list[list], num_tokens: int) -> set: + """Keys a cold consumer's lookup scan would demand for `num_tokens`. + + Drives the real scan functions with every key resolving as a promoting + secondary tier resolves it. + """ + scan = _make_scheduler_with_lookup({}, default=LookupResult.HIT_PENDING) + for group_config, offload_keys in zip( + runner.connector_scheduler.config.kv_group_configs, offload_keys_per_group + ): + num_chunks = num_tokens // group_config.tokens_per_chunk + keys = offload_keys[:num_chunks] + window = group_config.sliding_window_size_in_chunks + if window is None: + _maximal_lookup(scan, keys) + else: + scan._sliding_window_lookup(keys, window, _EMPTY_REQ_CTX) + return {lookup_call.args[0] for lookup_call in scan.manager.lookup.call_args_list} + + +@pytest.mark.parametrize("warmth", ["cold", "gpu_warm", "primary_warm"]) +@pytest.mark.parametrize("with_swa_group", [False, True]) +def test_request_level_supply_covers_consumer_demand( + request_runner, warmth: str, with_swa_group: bool +): + """A REQUEST_LEVEL producer must offer every key a consumer will demand. + + Same relation as `test_sliding_window_demand_is_store_reachable`, but at + the level where the key set is really built: `_build_store_jobs` prunes + chunks with a null GPU block and unreachable SWA chunks, while the demand + comes from an independent scan. + """ + full_attn_block_size = 16 + swa_block_size = 4 + num_tokens = 32 + + kv_cache_groups = ( + _full_and_swa_groups(full_attn_block_size, swa_block_size, sliding_window=8) + if with_swa_group + else None + ) + runner = request_runner( + block_size=swa_block_size if with_swa_group else full_attn_block_size, + num_gpu_blocks=200, + async_scheduling=False, + kv_cache_groups=kv_cache_groups, + ) + + def store_everything_offered(): + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + + # `_run` rather than `run`, which asserts on an expected GPU-block set this + # test deliberately does not hard-code. + if warmth != "cold": + runner.new_request(token_ids=[0] * num_tokens) + store_everything_offered() + runner._run(decoded_tokens=[0], complete_transfers=True) + runner._run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=True) + if warmth == "primary_warm": + # Warm only in the primary tier, so the producer must load first. + runner.scheduler.reset_prefix_cache() + + # The producer leg: same prompt, REQUEST_LEVEL, so prefix hits stay in the + # store path instead of being skipped. + runner.manager.reset_mock() + store_everything_offered() + runner.manager.on_new_request.return_value = RequestOffloadingContext( + policy=OffloadPolicy.REQUEST_LEVEL + ) + if warmth == "primary_warm": + # HIT is final; HIT_PENDING would defer the request on every step. + runner.manager.lookup.return_value = LookupResult.HIT + runner.new_request(token_ids=[0] * num_tokens) + + offload_keys_per_group: list[list] = [] + + def capture_offload_keys() -> None: + for req_status in runner.connector_scheduler._req_status.values(): + offload_keys_per_group.clear() + offload_keys_per_group.extend( + list(group_state.offload_keys) + for group_state in req_status.group_states + ) + + runner._run( + decoded_tokens=[0], + complete_transfers=True, + post_step_fn=capture_offload_keys, + ) + runner._run( + decoded_tokens=[EOS_TOKEN_ID], + complete_transfers=True, + post_step_fn=capture_offload_keys, + ) + + assert offload_keys_per_group, "no request state was captured" + supplied = { + key + for store_call in runner.manager.prepare_store.call_args_list + for key in store_call.args[0] + } + demanded = _demanded_keys(runner, offload_keys_per_group, num_tokens) + + assert demanded, "the scan must demand something" + missing = demanded - supplied + assert not missing, ( + f"{len(missing)} of {len(demanded)} demanded keys are never offered to " + f"prepare_store, so a peer fetching them would wait out the load timeout" + ) + + @pytest.mark.parametrize("async_scheduling", [True, False]) def test_stale_sliding_window_block_after_prepare_store_failure( request_runner, async_scheduling: bool diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index 3397a75c3c..b1bb96f5b6 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -65,7 +65,7 @@ def to_key(int_hash: int) -> OffloadKey: return make_offload_key(str(int_hash).encode(), 0) -def to_keys(int_hashes: list[int]) -> list[OffloadKey]: +def to_keys(int_hashes: Iterable[int]) -> list[OffloadKey]: return [to_key(i) for i in int_hashes] diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index 2cc9618303..e93eb77de3 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -17,7 +17,12 @@ import pytest from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_utils import DEFAULT_NONE_HASH_SEED, init_none_hash -from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext +from vllm.v1.kv_offload.base import ( + LookupResult, + OffloadPolicy, + ReqContext, + ScheduleEndContext, +) from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob from vllm.v1.kv_offload.tiering.p2p import manager as manager_module from vllm.v1.kv_offload.tiering.p2p.manager import ( @@ -240,6 +245,28 @@ class TestLookup: assert mgr.lookup(b"key", ctx) is LookupResult.MISS +# --------------------------------------------------------------------------- +# Tests for on_new_request offload policy +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "kv_params,expected", + [ + (_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL), + (None, OffloadPolicy.BLOCK_LEVEL), + (_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL), + ({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL), + ], + ids=["producer", "plain", "consumer", "producer_no_id"], +) +def test_on_new_request_policy(monkeypatch, kv_params, expected): + """Only a producer leg carrying a kv_request_id widens to REQUEST_LEVEL.""" + mgr = _make_manager() + monkeypatch.setattr(mgr, "_get_or_create_session", lambda peer_id: None) + assert mgr.on_new_request(_req_context(kv_params=kv_params)).policy is expected + + # --------------------------------------------------------------------------- # Tests for serve_external_requests # --------------------------------------------------------------------------- diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index a0e26293f0..88d7c466ca 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -1063,10 +1063,17 @@ class TestTieringOffloadingManager: self.manager.on_request_finished(ctx) assert ctx.req_id not in self.manager._req_state + @pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"]) def test_prepare_store_cascades_existing_blocks_to_request_level_tiers( - self, manager_setup + self, manager_setup, new_ids ): - """prepare_store cascades hit blocks to request-level tiers only.""" + """prepare_store cascades hit blocks to request-level tiers only. + + The fully-warm case is the producer shape from issue #52808: nothing is + new, so primary returns an empty keys_to_store and no GPU->primary + transfer runs. A request-level tier must still be handed the blocks, + otherwise a peer waiting on them has nothing to fetch. + """ # Store some blocks to primary first existing_blocks = to_keys(range(3)) self._start_request() @@ -1093,7 +1100,7 @@ class TestTieringOffloadingManager: ) # Call prepare_store with existing + new blocks - new_blocks = to_keys(range(3, 5)) + new_blocks = to_keys(new_ids) all_blocks = existing_blocks + new_blocks result = self.manager.prepare_store(all_blocks, ctx) assert result is not None diff --git a/vllm/v1/kv_offload/tiering/p2p/manager.py b/vllm/v1/kv_offload/tiering/p2p/manager.py index 8050be0f4f..9d8ab38c65 100644 --- a/vllm/v1/kv_offload/tiering/p2p/manager.py +++ b/vllm/v1/kv_offload/tiering/p2p/manager.py @@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import get_none_hash_seed from vllm.v1.kv_offload.base import ( LookupResult, OffloadKey, + OffloadPolicy, ReqContext, RequestOffloadingContext, ScheduleEndContext, @@ -361,11 +362,18 @@ class P2PSecondaryTierManager(SecondaryTierManager): prefiller side, sessions are created when the consumer's inbound connection arrives in _accept_new_peers — submit_store no longer pre-creates anything. + + Producer-leg requests (``remote_decoder`` present) ask for + REQUEST_LEVEL: the peer needs every block of the request, not just + the ones this request computed. """ _annotate_req_context(req_context) source = req_context.get_state(P2PSourceInfo) if source is not None: self._get_or_create_session(source.peer_id) + dest = req_context.get_state(P2PDestInfo) + if dest is not None and dest.kv_request_id: + return RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL) return RequestOffloadingContext() @override