[Bugfix][KVOffload] P2P tier declares REQUEST_LEVEL on the producer leg (#52912)

Signed-off-by: Liran Schour <lirans@il.ibm.com>
This commit is contained in:
liranschour
2026-08-31 10:07:20 +00:00
committed by GitHub
parent 699e180df4
commit d8de4ae322
5 changed files with 282 additions and 9 deletions
@@ -1293,12 +1293,21 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
def _make_scheduler_with_lookup(
lookup_results: dict[int, LookupResult],
default: LookupResult = LookupResult.MISS,
) -> OffloadingConnectorScheduler:
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup."""
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup.
Keys are addressed by their integer hash; `default` answers anything else.
"""
manager = MagicMock(spec=OffloadingManager)
manager.lookup.side_effect = lambda key, req_context: lookup_results.get(
int(get_offload_block_hash(key).decode()), LookupResult.MISS
)
def lookup(key, req_context):
block_hash = get_offload_block_hash(key)
if not block_hash.isdigit():
return default
return lookup_results.get(int(block_hash.decode()), default)
manager.lookup.side_effect = lookup
scheduler = object.__new__(OffloadingConnectorScheduler)
scheduler.manager = manager
@@ -1322,6 +1331,32 @@ def _maximal_lookup(sched, keys, start_chunk_idx: int = 0):
)
# Lookups issued, and end index returned, by a window-1 scan over 3 keys all
# resolving to the same result. A result that keeps the streak alive ends the
# scan at the first key; one that resets it makes the scan walk every key,
# which is what widens the demanded set. A new member has to be added here.
_SCAN_BEHAVIOR = {
LookupResult.HIT: (1, 3),
LookupResult.HIT_PENDING: (1, None),
LookupResult.RETRY: (3, None),
LookupResult.MISS: (3, 0),
}
@pytest.mark.parametrize("result", list(LookupResult))
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
"""Whether a result keeps a sliding-window streak alive decides how wide
the demanded chunk set gets, so every member needs deliberate behavior."""
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
keys = to_keys([1, 2, 3])
sched = _make_scheduler_with_lookup(dict.fromkeys([1, 2, 3], result))
assert sched._sliding_window_lookup(keys, 1, _EMPTY_REQ_CTX) == expected_end
assert len(sched.manager.lookup.call_args_list) == expected_lookups
class TestMaximalPrefixLookup:
def test_all_hit(self):
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
@@ -1556,6 +1591,62 @@ class TestSlidingWindowLookup:
)
# ---------------------------------------------------------------------------
# Tests for SWA store pruning vs. load demand
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64])
@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3])
@pytest.mark.parametrize("is_eagle_group", [False, True])
@pytest.mark.parametrize("tail_chunks", [0, 1, 5])
def test_sliding_window_demand_is_store_reachable(
alignment_chunk_count: int,
sliding_window_chunks: int,
is_eagle_group: bool,
tail_chunks: int,
):
"""Every chunk the load path looks up must be one the store path keeps.
`is_store_reachable_swa_chunk` prunes chunks `_sliding_window_lookup` can
never query, but the two mirror each other by hand: this pins the relation,
where `test_is_store_reachable_swa_chunk` pins only the predicate's values.
A looked-up key is a demanded key, since a secondary-tier hit queues a
promotion immediately.
"""
storable_chunk_count = alignment_chunk_count + tail_chunks
# Both sides widen by one for an unverified EAGLE group.
required_window = sliding_window_chunks + int(is_eagle_group)
sched = _make_scheduler_with_lookup(
{i: LookupResult.HIT_PENDING for i in range(storable_chunk_count)}
)
sched._sliding_window_lookup(
to_keys(range(storable_chunk_count)), required_window, _EMPTY_REQ_CTX
)
demanded = {
int(get_offload_block_hash(lookup_call.args[0]).decode())
for lookup_call in sched.manager.lookup.call_args_list
}
assert demanded, "the scan must query something"
unsuppliable = sorted(
chunk_idx
for chunk_idx in demanded
if not is_store_reachable_swa_chunk(
chunk_idx,
storable_chunk_count,
alignment_chunk_count,
sliding_window_chunks,
is_eagle_group,
)
)
assert not unsuppliable, (
f"chunks {unsuppliable} are demanded by the load path but pruned by the "
f"store path, so a warm producer can never supply them"
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
@@ -2308,6 +2399,146 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
)
def _full_and_swa_groups(
full_attn_block_size: int, swa_block_size: int, sliding_window: int
) -> list[KVCacheGroupSpec]:
"""Group 0 full attention (MLA-like), group 1 sliding window."""
return [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=full_attn_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=swa_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
),
]
def _demanded_keys(runner, offload_keys_per_group: list[list], num_tokens: int) -> set:
"""Keys a cold consumer's lookup scan would demand for `num_tokens`.
Drives the real scan functions with every key resolving as a promoting
secondary tier resolves it.
"""
scan = _make_scheduler_with_lookup({}, default=LookupResult.HIT_PENDING)
for group_config, offload_keys in zip(
runner.connector_scheduler.config.kv_group_configs, offload_keys_per_group
):
num_chunks = num_tokens // group_config.tokens_per_chunk
keys = offload_keys[:num_chunks]
window = group_config.sliding_window_size_in_chunks
if window is None:
_maximal_lookup(scan, keys)
else:
scan._sliding_window_lookup(keys, window, _EMPTY_REQ_CTX)
return {lookup_call.args[0] for lookup_call in scan.manager.lookup.call_args_list}
@pytest.mark.parametrize("warmth", ["cold", "gpu_warm", "primary_warm"])
@pytest.mark.parametrize("with_swa_group", [False, True])
def test_request_level_supply_covers_consumer_demand(
request_runner, warmth: str, with_swa_group: bool
):
"""A REQUEST_LEVEL producer must offer every key a consumer will demand.
Same relation as `test_sliding_window_demand_is_store_reachable`, but at
the level where the key set is really built: `_build_store_jobs` prunes
chunks with a null GPU block and unreachable SWA chunks, while the demand
comes from an independent scan.
"""
full_attn_block_size = 16
swa_block_size = 4
num_tokens = 32
kv_cache_groups = (
_full_and_swa_groups(full_attn_block_size, swa_block_size, sliding_window=8)
if with_swa_group
else None
)
runner = request_runner(
block_size=swa_block_size if with_swa_group else full_attn_block_size,
num_gpu_blocks=200,
async_scheduling=False,
kv_cache_groups=kv_cache_groups,
)
def store_everything_offered():
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
# `_run` rather than `run`, which asserts on an expected GPU-block set this
# test deliberately does not hard-code.
if warmth != "cold":
runner.new_request(token_ids=[0] * num_tokens)
store_everything_offered()
runner._run(decoded_tokens=[0], complete_transfers=True)
runner._run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=True)
if warmth == "primary_warm":
# Warm only in the primary tier, so the producer must load first.
runner.scheduler.reset_prefix_cache()
# The producer leg: same prompt, REQUEST_LEVEL, so prefix hits stay in the
# store path instead of being skipped.
runner.manager.reset_mock()
store_everything_offered()
runner.manager.on_new_request.return_value = RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
if warmth == "primary_warm":
# HIT is final; HIT_PENDING would defer the request on every step.
runner.manager.lookup.return_value = LookupResult.HIT
runner.new_request(token_ids=[0] * num_tokens)
offload_keys_per_group: list[list] = []
def capture_offload_keys() -> None:
for req_status in runner.connector_scheduler._req_status.values():
offload_keys_per_group.clear()
offload_keys_per_group.extend(
list(group_state.offload_keys)
for group_state in req_status.group_states
)
runner._run(
decoded_tokens=[0],
complete_transfers=True,
post_step_fn=capture_offload_keys,
)
runner._run(
decoded_tokens=[EOS_TOKEN_ID],
complete_transfers=True,
post_step_fn=capture_offload_keys,
)
assert offload_keys_per_group, "no request state was captured"
supplied = {
key
for store_call in runner.manager.prepare_store.call_args_list
for key in store_call.args[0]
}
demanded = _demanded_keys(runner, offload_keys_per_group, num_tokens)
assert demanded, "the scan must demand something"
missing = demanded - supplied
assert not missing, (
f"{len(missing)} of {len(demanded)} demanded keys are never offered to "
f"prepare_store, so a peer fetching them would wait out the load timeout"
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_stale_sliding_window_block_after_prepare_store_failure(
request_runner, async_scheduling: bool
@@ -65,7 +65,7 @@ def to_key(int_hash: int) -> OffloadKey:
return make_offload_key(str(int_hash).encode(), 0)
def to_keys(int_hashes: list[int]) -> list[OffloadKey]:
def to_keys(int_hashes: Iterable[int]) -> list[OffloadKey]:
return [to_key(i) for i in int_hashes]
@@ -17,7 +17,12 @@ import pytest
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import DEFAULT_NONE_HASH_SEED, init_none_hash
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadPolicy,
ReqContext,
ScheduleEndContext,
)
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
from vllm.v1.kv_offload.tiering.p2p.manager import (
@@ -240,6 +245,28 @@ class TestLookup:
assert mgr.lookup(b"key", ctx) is LookupResult.MISS
# ---------------------------------------------------------------------------
# Tests for on_new_request offload policy
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"kv_params,expected",
[
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
(None, OffloadPolicy.BLOCK_LEVEL),
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
],
ids=["producer", "plain", "consumer", "producer_no_id"],
)
def test_on_new_request_policy(monkeypatch, kv_params, expected):
"""Only a producer leg carrying a kv_request_id widens to REQUEST_LEVEL."""
mgr = _make_manager()
monkeypatch.setattr(mgr, "_get_or_create_session", lambda peer_id: None)
assert mgr.on_new_request(_req_context(kv_params=kv_params)).policy is expected
# ---------------------------------------------------------------------------
# Tests for serve_external_requests
# ---------------------------------------------------------------------------
@@ -1063,10 +1063,17 @@ class TestTieringOffloadingManager:
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._req_state
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
self, manager_setup
self, manager_setup, new_ids
):
"""prepare_store cascades hit blocks to request-level tiers only."""
"""prepare_store cascades hit blocks to request-level tiers only.
The fully-warm case is the producer shape from issue #52808: nothing is
new, so primary returns an empty keys_to_store and no GPU->primary
transfer runs. A request-level tier must still be handed the blocks,
otherwise a peer waiting on them has nothing to fetch.
"""
# Store some blocks to primary first
existing_blocks = to_keys(range(3))
self._start_request()
@@ -1093,7 +1100,7 @@ class TestTieringOffloadingManager:
)
# Call prepare_store with existing + new blocks
new_blocks = to_keys(range(3, 5))
new_blocks = to_keys(new_ids)
all_blocks = existing_blocks + new_blocks
result = self.manager.prepare_store(all_blocks, ctx)
assert result is not None
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import get_none_hash_seed
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
@@ -361,11 +362,18 @@ class P2PSecondaryTierManager(SecondaryTierManager):
prefiller side, sessions are created when the consumer's inbound
connection arrives in _accept_new_peers — submit_store no longer
pre-creates anything.
Producer-leg requests (``remote_decoder`` present) ask for
REQUEST_LEVEL: the peer needs every block of the request, not just
the ones this request computed.
"""
_annotate_req_context(req_context)
source = req_context.get_state(P2PSourceInfo)
if source is not None:
self._get_or_create_session(source.peer_id)
dest = req_context.get_state(P2PDestInfo)
if dest is not None and dest.kv_request_id:
return RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
return RequestOffloadingContext()
@override