[Bugfix][KVOffload] P2P tier declares REQUEST_LEVEL on the producer leg (#52912)
Signed-off-by: Liran Schour <lirans@il.ibm.com>
This commit is contained in:
@@ -1293,12 +1293,21 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
|
||||
def _make_scheduler_with_lookup(
|
||||
lookup_results: dict[int, LookupResult],
|
||||
default: LookupResult = LookupResult.MISS,
|
||||
) -> OffloadingConnectorScheduler:
|
||||
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup."""
|
||||
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup.
|
||||
|
||||
Keys are addressed by their integer hash; `default` answers anything else.
|
||||
"""
|
||||
manager = MagicMock(spec=OffloadingManager)
|
||||
manager.lookup.side_effect = lambda key, req_context: lookup_results.get(
|
||||
int(get_offload_block_hash(key).decode()), LookupResult.MISS
|
||||
)
|
||||
|
||||
def lookup(key, req_context):
|
||||
block_hash = get_offload_block_hash(key)
|
||||
if not block_hash.isdigit():
|
||||
return default
|
||||
return lookup_results.get(int(block_hash.decode()), default)
|
||||
|
||||
manager.lookup.side_effect = lookup
|
||||
|
||||
scheduler = object.__new__(OffloadingConnectorScheduler)
|
||||
scheduler.manager = manager
|
||||
@@ -1322,6 +1331,32 @@ def _maximal_lookup(sched, keys, start_chunk_idx: int = 0):
|
||||
)
|
||||
|
||||
|
||||
# Lookups issued, and end index returned, by a window-1 scan over 3 keys all
|
||||
# resolving to the same result. A result that keeps the streak alive ends the
|
||||
# scan at the first key; one that resets it makes the scan walk every key,
|
||||
# which is what widens the demanded set. A new member has to be added here.
|
||||
_SCAN_BEHAVIOR = {
|
||||
LookupResult.HIT: (1, 3),
|
||||
LookupResult.HIT_PENDING: (1, None),
|
||||
LookupResult.RETRY: (3, None),
|
||||
LookupResult.MISS: (3, 0),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", list(LookupResult))
|
||||
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
|
||||
"""Whether a result keeps a sliding-window streak alive decides how wide
|
||||
the demanded chunk set gets, so every member needs deliberate behavior."""
|
||||
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
|
||||
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
|
||||
|
||||
keys = to_keys([1, 2, 3])
|
||||
sched = _make_scheduler_with_lookup(dict.fromkeys([1, 2, 3], result))
|
||||
|
||||
assert sched._sliding_window_lookup(keys, 1, _EMPTY_REQ_CTX) == expected_end
|
||||
assert len(sched.manager.lookup.call_args_list) == expected_lookups
|
||||
|
||||
|
||||
class TestMaximalPrefixLookup:
|
||||
def test_all_hit(self):
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
||||
@@ -1556,6 +1591,62 @@ class TestSlidingWindowLookup:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for SWA store pruning vs. load demand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64])
|
||||
@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3])
|
||||
@pytest.mark.parametrize("is_eagle_group", [False, True])
|
||||
@pytest.mark.parametrize("tail_chunks", [0, 1, 5])
|
||||
def test_sliding_window_demand_is_store_reachable(
|
||||
alignment_chunk_count: int,
|
||||
sliding_window_chunks: int,
|
||||
is_eagle_group: bool,
|
||||
tail_chunks: int,
|
||||
):
|
||||
"""Every chunk the load path looks up must be one the store path keeps.
|
||||
|
||||
`is_store_reachable_swa_chunk` prunes chunks `_sliding_window_lookup` can
|
||||
never query, but the two mirror each other by hand: this pins the relation,
|
||||
where `test_is_store_reachable_swa_chunk` pins only the predicate's values.
|
||||
A looked-up key is a demanded key, since a secondary-tier hit queues a
|
||||
promotion immediately.
|
||||
"""
|
||||
storable_chunk_count = alignment_chunk_count + tail_chunks
|
||||
# Both sides widen by one for an unverified EAGLE group.
|
||||
required_window = sliding_window_chunks + int(is_eagle_group)
|
||||
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{i: LookupResult.HIT_PENDING for i in range(storable_chunk_count)}
|
||||
)
|
||||
sched._sliding_window_lookup(
|
||||
to_keys(range(storable_chunk_count)), required_window, _EMPTY_REQ_CTX
|
||||
)
|
||||
|
||||
demanded = {
|
||||
int(get_offload_block_hash(lookup_call.args[0]).decode())
|
||||
for lookup_call in sched.manager.lookup.call_args_list
|
||||
}
|
||||
assert demanded, "the scan must query something"
|
||||
unsuppliable = sorted(
|
||||
chunk_idx
|
||||
for chunk_idx in demanded
|
||||
if not is_store_reachable_swa_chunk(
|
||||
chunk_idx,
|
||||
storable_chunk_count,
|
||||
alignment_chunk_count,
|
||||
sliding_window_chunks,
|
||||
is_eagle_group,
|
||||
)
|
||||
)
|
||||
assert not unsuppliable, (
|
||||
f"chunks {unsuppliable} are demanded by the load path but pruned by the "
|
||||
f"store path, so a warm producer can never supply them"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
|
||||
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
|
||||
@@ -2308,6 +2399,146 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
)
|
||||
|
||||
|
||||
def _full_and_swa_groups(
|
||||
full_attn_block_size: int, swa_block_size: int, sliding_window: int
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""Group 0 full attention (MLA-like), group 1 sliding window."""
|
||||
return [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=full_attn_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
SlidingWindowSpec(
|
||||
block_size=swa_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _demanded_keys(runner, offload_keys_per_group: list[list], num_tokens: int) -> set:
|
||||
"""Keys a cold consumer's lookup scan would demand for `num_tokens`.
|
||||
|
||||
Drives the real scan functions with every key resolving as a promoting
|
||||
secondary tier resolves it.
|
||||
"""
|
||||
scan = _make_scheduler_with_lookup({}, default=LookupResult.HIT_PENDING)
|
||||
for group_config, offload_keys in zip(
|
||||
runner.connector_scheduler.config.kv_group_configs, offload_keys_per_group
|
||||
):
|
||||
num_chunks = num_tokens // group_config.tokens_per_chunk
|
||||
keys = offload_keys[:num_chunks]
|
||||
window = group_config.sliding_window_size_in_chunks
|
||||
if window is None:
|
||||
_maximal_lookup(scan, keys)
|
||||
else:
|
||||
scan._sliding_window_lookup(keys, window, _EMPTY_REQ_CTX)
|
||||
return {lookup_call.args[0] for lookup_call in scan.manager.lookup.call_args_list}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("warmth", ["cold", "gpu_warm", "primary_warm"])
|
||||
@pytest.mark.parametrize("with_swa_group", [False, True])
|
||||
def test_request_level_supply_covers_consumer_demand(
|
||||
request_runner, warmth: str, with_swa_group: bool
|
||||
):
|
||||
"""A REQUEST_LEVEL producer must offer every key a consumer will demand.
|
||||
|
||||
Same relation as `test_sliding_window_demand_is_store_reachable`, but at
|
||||
the level where the key set is really built: `_build_store_jobs` prunes
|
||||
chunks with a null GPU block and unreachable SWA chunks, while the demand
|
||||
comes from an independent scan.
|
||||
"""
|
||||
full_attn_block_size = 16
|
||||
swa_block_size = 4
|
||||
num_tokens = 32
|
||||
|
||||
kv_cache_groups = (
|
||||
_full_and_swa_groups(full_attn_block_size, swa_block_size, sliding_window=8)
|
||||
if with_swa_group
|
||||
else None
|
||||
)
|
||||
runner = request_runner(
|
||||
block_size=swa_block_size if with_swa_group else full_attn_block_size,
|
||||
num_gpu_blocks=200,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
def store_everything_offered():
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
# `_run` rather than `run`, which asserts on an expected GPU-block set this
|
||||
# test deliberately does not hard-code.
|
||||
if warmth != "cold":
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
store_everything_offered()
|
||||
runner._run(decoded_tokens=[0], complete_transfers=True)
|
||||
runner._run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=True)
|
||||
if warmth == "primary_warm":
|
||||
# Warm only in the primary tier, so the producer must load first.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
# The producer leg: same prompt, REQUEST_LEVEL, so prefix hits stay in the
|
||||
# store path instead of being skipped.
|
||||
runner.manager.reset_mock()
|
||||
store_everything_offered()
|
||||
runner.manager.on_new_request.return_value = RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
if warmth == "primary_warm":
|
||||
# HIT is final; HIT_PENDING would defer the request on every step.
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
|
||||
offload_keys_per_group: list[list] = []
|
||||
|
||||
def capture_offload_keys() -> None:
|
||||
for req_status in runner.connector_scheduler._req_status.values():
|
||||
offload_keys_per_group.clear()
|
||||
offload_keys_per_group.extend(
|
||||
list(group_state.offload_keys)
|
||||
for group_state in req_status.group_states
|
||||
)
|
||||
|
||||
runner._run(
|
||||
decoded_tokens=[0],
|
||||
complete_transfers=True,
|
||||
post_step_fn=capture_offload_keys,
|
||||
)
|
||||
runner._run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
complete_transfers=True,
|
||||
post_step_fn=capture_offload_keys,
|
||||
)
|
||||
|
||||
assert offload_keys_per_group, "no request state was captured"
|
||||
supplied = {
|
||||
key
|
||||
for store_call in runner.manager.prepare_store.call_args_list
|
||||
for key in store_call.args[0]
|
||||
}
|
||||
demanded = _demanded_keys(runner, offload_keys_per_group, num_tokens)
|
||||
|
||||
assert demanded, "the scan must demand something"
|
||||
missing = demanded - supplied
|
||||
assert not missing, (
|
||||
f"{len(missing)} of {len(demanded)} demanded keys are never offered to "
|
||||
f"prepare_store, so a peer fetching them would wait out the load timeout"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
request_runner, async_scheduling: bool
|
||||
|
||||
@@ -65,7 +65,7 @@ def to_key(int_hash: int) -> OffloadKey:
|
||||
return make_offload_key(str(int_hash).encode(), 0)
|
||||
|
||||
|
||||
def to_keys(int_hashes: list[int]) -> list[OffloadKey]:
|
||||
def to_keys(int_hashes: Iterable[int]) -> list[OffloadKey]:
|
||||
return [to_key(i) for i in int_hashes]
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,12 @@ import pytest
|
||||
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import DEFAULT_NONE_HASH_SEED, init_none_hash
|
||||
from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
ScheduleEndContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobResult, TransferJob
|
||||
from vllm.v1.kv_offload.tiering.p2p import manager as manager_module
|
||||
from vllm.v1.kv_offload.tiering.p2p.manager import (
|
||||
@@ -240,6 +245,28 @@ class TestLookup:
|
||||
assert mgr.lookup(b"key", ctx) is LookupResult.MISS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for on_new_request offload policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_params,expected",
|
||||
[
|
||||
(_remote_decoder_kv_params(), OffloadPolicy.REQUEST_LEVEL),
|
||||
(None, OffloadPolicy.BLOCK_LEVEL),
|
||||
(_remote_prefiller_kv_params(), OffloadPolicy.BLOCK_LEVEL),
|
||||
({"remote_decoder": {}}, OffloadPolicy.BLOCK_LEVEL),
|
||||
],
|
||||
ids=["producer", "plain", "consumer", "producer_no_id"],
|
||||
)
|
||||
def test_on_new_request_policy(monkeypatch, kv_params, expected):
|
||||
"""Only a producer leg carrying a kv_request_id widens to REQUEST_LEVEL."""
|
||||
mgr = _make_manager()
|
||||
monkeypatch.setattr(mgr, "_get_or_create_session", lambda peer_id: None)
|
||||
assert mgr.on_new_request(_req_context(kv_params=kv_params)).policy is expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for serve_external_requests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1063,10 +1063,17 @@ class TestTieringOffloadingManager:
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
@pytest.mark.parametrize("new_ids", [(), (3, 4)], ids=["fully_warm", "mixed"])
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
self, manager_setup
|
||||
self, manager_setup, new_ids
|
||||
):
|
||||
"""prepare_store cascades hit blocks to request-level tiers only."""
|
||||
"""prepare_store cascades hit blocks to request-level tiers only.
|
||||
|
||||
The fully-warm case is the producer shape from issue #52808: nothing is
|
||||
new, so primary returns an empty keys_to_store and no GPU->primary
|
||||
transfer runs. A request-level tier must still be handed the blocks,
|
||||
otherwise a peer waiting on them has nothing to fetch.
|
||||
"""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
@@ -1093,7 +1100,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Call prepare_store with existing + new blocks
|
||||
new_blocks = to_keys(range(3, 5))
|
||||
new_blocks = to_keys(new_ids)
|
||||
all_blocks = existing_blocks + new_blocks
|
||||
result = self.manager.prepare_store(all_blocks, ctx)
|
||||
assert result is not None
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import get_none_hash_seed
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
ScheduleEndContext,
|
||||
@@ -361,11 +362,18 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
prefiller side, sessions are created when the consumer's inbound
|
||||
connection arrives in _accept_new_peers — submit_store no longer
|
||||
pre-creates anything.
|
||||
|
||||
Producer-leg requests (``remote_decoder`` present) ask for
|
||||
REQUEST_LEVEL: the peer needs every block of the request, not just
|
||||
the ones this request computed.
|
||||
"""
|
||||
_annotate_req_context(req_context)
|
||||
source = req_context.get_state(P2PSourceInfo)
|
||||
if source is not None:
|
||||
self._get_or_create_session(source.peer_id)
|
||||
dest = req_context.get_state(P2PDestInfo)
|
||||
if dest is not None and dest.kv_request_id:
|
||||
return RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
|
||||
Reference in New Issue
Block a user