[KV Offloading] Replace bool|None lookup return with LookupResult enum (#46363)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Ronen Schaffer
2026-06-24 18:06:08 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 7f99e80c3b
commit bb61177e49
14 changed files with 323 additions and 172 deletions
+29 -28
View File
@@ -8,6 +8,7 @@ import pytest
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
OffloadingEvent,
OffloadKey,
PrepareStoreOutput,
@@ -160,7 +161,7 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
# block 2 must still be present in the cache
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
def test_filter_reused_manager_reports_stores_skipped_counter():
@@ -242,8 +243,8 @@ def test_cpu_manager():
)
# lookup [1, 2] -> write in-flight, not yet ready
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is None
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is None
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.HIT_PENDING
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT_PENDING
# no events so far
assert list(cpu_manager.take_events()) == []
@@ -253,9 +254,9 @@ def test_cpu_manager():
verify_events(cpu_manager.take_events(), expected_stores=({1, 2},))
# lookup [1, 2]
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare store [2, 3, 4, 5] -> evicts [1]
prepare_store_output = cpu_manager.prepare_store(
@@ -280,12 +281,12 @@ def test_cpu_manager():
cpu_manager.complete_store(to_keys([2, 3, 4, 5]), _EMPTY_REQ_CTX)
# lookup (now that we have [2, 3, 4, 5])
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(4), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.MISS
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(4), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare load [2, 3]
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
@@ -329,8 +330,8 @@ def test_cpu_manager():
cpu_manager.complete_store(to_keys([7, 9]), _EMPTY_REQ_CTX, success=False)
# assert [7] is still stored, but [9] is not
assert cpu_manager.lookup(to_key(7), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(9), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(7), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(9), _EMPTY_REQ_CTX) is LookupResult.MISS
verify_events(
cpu_manager.take_events(),
@@ -412,8 +413,8 @@ class TestARCPolicy:
)
# lookup [1, 2] -> write in-flight, not yet ready
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is None
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is None
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.HIT_PENDING
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT_PENDING
# no events so far
assert list(cpu_manager.take_events()) == []
@@ -423,9 +424,9 @@ class TestARCPolicy:
verify_events(cpu_manager.take_events(), expected_stores=({1, 2},))
# lookup [1, 2]
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
# blocks should be in T1 (recent)
assert len(arc_policy.t1) == 2
@@ -629,7 +630,7 @@ class TestARCPolicy:
cpu_manager.complete_store(to_keys([5]), _EMPTY_REQ_CTX, success=False)
# block 5 should not be in cache
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is False
assert cpu_manager.lookup(to_key(5), _EMPTY_REQ_CTX) is LookupResult.MISS
# block 5 should not be in T1 or T2
assert to_keys([5])[0] not in arc_policy.t1
assert to_keys([5])[0] not in arc_policy.t2
@@ -670,8 +671,8 @@ class TestARCPolicy:
cpu_manager.complete_store(to_keys([6]), _EMPTY_REQ_CTX)
# verify blocks 2, 3 (in T2) are still present
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is True
assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.HIT
assert cpu_manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.HIT
# verify events
events = list(cpu_manager.take_events())
@@ -691,8 +692,8 @@ def test_filter_reused_manager():
)
# Lookup [1, 2] -> 1st time, added to tracker but not eligible for store yet
assert manager.lookup(to_key(1), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.MISS
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare store [1, 2] -> should be filtered
prepare_store_output = manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -700,7 +701,7 @@ def test_filter_reused_manager():
assert prepare_store_output.keys_to_store == []
# Lookup [1] -> 2nd time, eligible now
assert manager.lookup(to_key(1), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(1), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare store [1, 2] -> [1] should be eligible, [2] should be filtered
prepare_store_output = manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
@@ -709,13 +710,13 @@ def test_filter_reused_manager():
# Lookup [3, 4] -> 1st time
# (evicts [2] from tracker since max_size is 3 and tracker has [1])
assert manager.lookup(to_key(3), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(4), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(3), _EMPTY_REQ_CTX) is LookupResult.MISS
assert manager.lookup(to_key(4), _EMPTY_REQ_CTX) is LookupResult.MISS
# Verify [2] was evicted from the tracker (tracker now has: [1], [3], [4])
assert to_keys([2])[0] not in manager.counts
# Lookup [2] again -> (this adds [2] back to the tracker as 1st time)
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is False
assert manager.lookup(to_key(2), _EMPTY_REQ_CTX) is LookupResult.MISS
# Verify [2] was re-added with count=1 (not eligible yet)
assert manager.counts.get(to_keys([2])[0]) == 1
+20 -6
View File
@@ -18,7 +18,12 @@ import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
ReqContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
@@ -166,7 +171,7 @@ def fs_tier(tmp_path):
def test_lookup_empty_tier(fs_tier):
tier, _ = fs_tier
results = lookup_and_wait(tier, [key(1), key(2)])
assert results == [False, False]
assert results == [LookupResult.MISS, LookupResult.MISS]
def test_store_creates_file_and_lookup_succeeds(fs_tier):
@@ -176,7 +181,7 @@ def test_store_creates_file_and_lookup_succeeds(fs_tier):
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert lookup_and_wait(tier, [key(1)]) == [True]
assert lookup_and_wait(tier, [key(1)]) == [LookupResult.HIT]
dest = tier.file_mapper.get_file_name(key(1))
assert os.path.exists(dest), f"Expected file at {dest}"
@@ -188,14 +193,20 @@ def test_store_then_load_roundtrip(fs_tier):
store_results = drain(tier)
assert all(r.success for r in store_results)
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
assert lookup_and_wait(tier, [key(1), key(2)]) == [
LookupResult.HIT,
LookupResult.HIT,
]
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
tier.submit_load(job_l)
load_results = drain(tier)
assert all(r.success for r in load_results)
# Blocks stay on disk after load
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
assert lookup_and_wait(tier, [key(1), key(2)]) == [
LookupResult.HIT,
LookupResult.HIT,
]
def test_invalid_path_raises_at_construction():
@@ -231,7 +242,10 @@ def test_multiple_jobs_tracked_independently(fs_tier):
results = drain(tier)
job_ids = {r.job_id for r in results}
assert job_ids == {1, 2}
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
assert lookup_and_wait(tier, [key(1), key(2)]) == [
LookupResult.HIT,
LookupResult.HIT,
]
def test_multi_block_job_partial_failure(fs_tier):
+16 -7
View File
@@ -17,7 +17,12 @@ from unittest.mock import MagicMock, patch
import numpy as np
import torch
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
ReqContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult
from vllm.v1.kv_offload.tiering.obj.manager import ObjectStoreSecondaryTierManager
@@ -236,19 +241,19 @@ class TestMockObjTierBasic:
self.tier, self.agent = _make_tier(num_blocks=4)
def test_lookup_empty_tier(self):
assert lookup_and_wait(self.tier, [key(1)]) == [False]
assert lookup_and_wait(self.tier, [key(1)]) == [LookupResult.MISS]
def test_store_and_lookup(self):
self.tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(self.tier)
assert len(results) == 1
assert results[0].success
assert lookup_and_wait(self.tier, [key(1)]) == [True]
assert lookup_and_wait(self.tier, [key(1)]) == [LookupResult.HIT]
def test_lookup_unrelated_key_returns_false(self):
self.tier.submit_store(make_job(1, [key(1)], [0]))
drain(self.tier)
assert lookup_and_wait(self.tier, [key(999)]) == [False]
assert lookup_and_wait(self.tier, [key(999)]) == [LookupResult.MISS]
def test_store_then_load_roundtrip(self):
self.tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
@@ -327,13 +332,17 @@ class TestMockObjTierMultiBlock:
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert lookup_and_wait(tier, keys) == [True] * 8
assert lookup_and_wait(tier, keys) == [LookupResult.HIT] * 8
def test_partial_block_lookup(self):
tier, _ = _make_tier(num_blocks=4)
tier.submit_store(make_job(1, [key(0), key(1)], [0, 1]))
drain(tier)
assert lookup_and_wait(tier, [key(0), key(1), key(2)]) == [True, True, False]
assert lookup_and_wait(tier, [key(0), key(1), key(2)]) == [
LookupResult.HIT,
LookupResult.HIT,
LookupResult.MISS,
]
class TestMockObjTierFailures:
@@ -342,7 +351,7 @@ class TestMockObjTierFailures:
agent.query_memory = lambda *a, **k: (_ for _ in ()).throw(
RuntimeError("backend error")
)
assert lookup_and_wait(tier, [key(1)]) == [False]
assert lookup_and_wait(tier, [key(1)]) == [LookupResult.MISS]
def test_submit_store_register_memory_failure_reported_in_get_finished(self):
tier, agent = _make_tier(num_blocks=4)
@@ -21,6 +21,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingCounterMetadata,
OffloadKey,
OffloadPolicy,
@@ -60,15 +61,15 @@ def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]:
def count_hits(manager, keys: list[OffloadKey]) -> int | None:
"""Count consecutive lookup hits from the start of keys.
Returns the count of leading True results, or None if any lookup
returns None (retry-later signal).
Returns the count of leading HIT results, or None if any lookup
returns HIT_PENDING or RETRY.
"""
count = 0
for key in keys:
result = manager.lookup(key, _CTX)
if result is None:
if result in (LookupResult.HIT_PENDING, LookupResult.RETRY):
return None
if not result:
if result is not LookupResult.HIT:
break
count += 1
return count
@@ -185,18 +186,18 @@ class TestExampleSecondaryTierManager:
# Initially empty
blocks = to_keys(range(3))
assert tier.lookup(blocks[0], _CTX) is False
assert tier.lookup(blocks[0], _CTX) is LookupResult.MISS
# Store blocks (simulate with direct insertion for testing)
tier.blocks[blocks[0]] = True
tier.blocks[blocks[1]] = True
# Lookup should find first two blocks
assert tier.lookup(blocks[0], _CTX) is True
assert tier.lookup(blocks[1], _CTX) is True
assert tier.lookup(blocks[0], _CTX) is LookupResult.HIT
assert tier.lookup(blocks[1], _CTX) is LookupResult.HIT
# Third block not present
assert tier.lookup(blocks[2], _CTX) is False
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
class TestTieringOffloadingManager:
@@ -283,8 +284,12 @@ class TestTieringOffloadingManager:
assert self.secondary_tier2.get_num_blocks() == 3
# Verify blocks are present
assert all(self.secondary_tier1.lookup(b, _CTX) for b in blocks)
assert all(self.secondary_tier2.lookup(b, _CTX) for b in blocks)
assert all(
self.secondary_tier1.lookup(b, _CTX) is LookupResult.HIT for b in blocks
)
assert all(
self.secondary_tier2.lookup(b, _CTX) is LookupResult.HIT for b in blocks
)
def test_ref_cnt_protection_during_cascade(self, manager_setup):
"""Test that ref_cnt protects blocks during cascade."""
@@ -355,7 +360,7 @@ class TestTieringOffloadingManager:
# Lookup each block to initiate promotion for all of them
for block in blocks:
result = self.manager.lookup(block, _CTX)
assert result is None # Retry later (promotion initiated)
assert result is LookupResult.RETRY # promotion initiated
# End of step 1: flushes deferred submit_load() calls
self._simulate_on_schedule_end()
@@ -470,11 +475,11 @@ class TestTieringOffloadingManager:
ctx_a = ReqContext(req_id="req_a")
ctx_b = ReqContext(req_id="req_b")
# All lookups return None: secondary hit triggers promotion (in-flight)
assert self.manager.lookup(blocks[0], ctx_a) is None
assert self.manager.lookup(blocks[1], ctx_a) is None
assert self.manager.lookup(blocks[2], ctx_b) is None
assert self.manager.lookup(blocks[3], ctx_b) is None
# All lookups return RETRY: secondary hit triggers promotion
assert self.manager.lookup(blocks[0], ctx_a) is LookupResult.RETRY
assert self.manager.lookup(blocks[1], ctx_a) is LookupResult.RETRY
assert self.manager.lookup(blocks[2], ctx_b) is LookupResult.RETRY
assert self.manager.lookup(blocks[3], ctx_b) is LookupResult.RETRY
# submit_load must not fire during lookup - only at end of step
self.secondary_tier1.submit_load.assert_not_called()
@@ -511,9 +516,10 @@ class TestTieringOffloadingManager:
result_a = self.manager.lookup(shared_block, ctx_a)
result_b = self.manager.lookup(shared_block, ctx_b)
# Both see None (in-flight), but promotion is only queued once
assert result_a is None
assert result_b is None
# First lookup triggers promotion (RETRY), second finds block
# already in primary with write in-flight (HIT_PENDING).
assert result_a is LookupResult.RETRY
assert result_b is LookupResult.HIT_PENDING
self._simulate_on_schedule_end()
@@ -796,7 +802,10 @@ class TestTieringOffloadingManager:
# the lookup that staged it).
promo_block = to_keys([99])[0]
self.secondary_tier1.blocks[promo_block] = True
assert self.manager.lookup(promo_block, ReqContext(req_id="pending")) is None
assert (
self.manager.lookup(promo_block, ReqContext(req_id="pending"))
is LookupResult.RETRY
)
assert self.manager._pending_load_submissions
# Request-level tier registration.
@@ -829,7 +838,7 @@ class TestTieringOffloadingManager:
assert self.primary_tier._num_allocated_blocks == 0
assert self.primary_tier._free_list == []
for block in blocks:
assert self.primary_tier.lookup(block, _CTX) is False
assert self.primary_tier.lookup(block, _CTX) is LookupResult.MISS
# Pending submission was dropped, not submitted.
self.secondary_tier1.submit_load.assert_not_called()