[3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Dao007forever
2026-07-26 23:22:17 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 544cb724c8
commit d742856610
13 changed files with 1448 additions and 49 deletions
@@ -223,6 +223,299 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
assert moved[0].block_hash_num_tokens == 6
def test_take_partial_tail_offloads_returns_cow_target():
"""The connector offload hand-off exposes the mamba CoW *target* block Y
(the durable boundary state), not the overwritten source X, and only at
the CoW step."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A registered the partial tail but has not CoW'd yet: no offload.
assert manager.take_partial_tail_offloads() == {}
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert source_block is not None
source_block_id = source_block[0].block_id
# Step B: the producer continues, triggering the CoW X->Y.
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
offloads = manager.take_partial_tail_offloads()
assert list(offloads.keys()) == ["0"]
assert len(offloads["0"]) == 1
group_id, block_id, boundary_tokens = offloads["0"][0]
assert group_id == 1 # the mamba group
assert boundary_tokens == 6
copies, _ = manager.take_kv_cache_block_copies()
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
# The offload points at the durable CoW target Y, not the overwritten X.
assert block_id == cow_copy.dst_block_id
assert block_id != source_block_id
# Draining clears it.
assert manager.take_partial_tail_offloads() == {}
# The hand-off pinned Y (its CoW retention is released after this step,
# and Y is off the request block table); freeing the request unpins it.
cow_block = manager.block_pool.blocks[block_id]
pinned_ref = cow_block.ref_cnt
assert pinned_ref >= 1
manager.free(req0)
assert cow_block.ref_cnt == pinned_ref - 1
def test_partial_tail_pin_survives_released_cow_retention():
"""If the CoW retention is released before the hand-off is drained
(immediate-free mode), the drain must rescue the cow block from the free
queue: a raw ref increment would leave a ref>0 block allocatable, and the
next allocation would pop it and assert."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# Retention released before the drain (defer_block_free=False ordering).
_copies, retained = manager.take_kv_cache_block_copies()
manager.block_pool.free_blocks(retained)
offloads = manager.take_partial_tail_offloads()
((_group_id, block_id, boundary_tokens),) = offloads["0"]
assert boundary_tokens == 6
cow_block = manager.block_pool.blocks[block_id]
assert cow_block.ref_cnt == 1
# The pinned block is out of the free queue: draining every free block
# neither trips the allocator's ref_cnt assert nor hands it out.
new_blocks = manager.block_pool.get_new_blocks(
manager.block_pool.get_num_free_blocks()
)
assert block_id not in {b.block_id for b in new_blocks}
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# The request dies (preempt/abort) before the scheduler drains.
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
assert manager.take_partial_tail_offloads() == {}
def test_take_partial_tail_offloads_empty_without_partial_tail():
"""A prompt ending on a block boundary registers no partial tail, so there
is nothing to offload."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
# 4-token prompt ends exactly on the mamba block boundary (block_size=4).
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
assert manager.take_partial_tail_offloads() == {}
req0.num_computed_tokens = 4
req0.append_output_token_ids([2])
assert manager.allocate_slots(req0, 1) is not None
assert manager.take_partial_tail_offloads() == {}
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
"""truncate_computed_blocks slices each group by its own block size,
keeps null placeholders in the retained prefix, and leaves the original
lookup result untouched (pure view, no refcount changes)."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=2 * hash_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
manager.new_step_starts()
consumer = make_request(
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
)
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
assert [len(group) for group in blocks.blocks] == [3, 2]
assert blocks.blocks[1][0].is_null
truncated = manager.truncate_computed_blocks(blocks, 4)
assert [len(group) for group in truncated.blocks] == [2, 1]
assert truncated.blocks[1][0].is_null
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size = 2
block_size = 2 * hash_block_size
@@ -3,6 +3,8 @@
from math import lcm
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
@@ -14,10 +16,20 @@ from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
)
def _mamba_align(block_size=32):
return MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
@@ -196,6 +208,49 @@ def test_coordinator_group_block_size_double_hash():
assert hit % 32 == 0
# ----- Fine-grained partial hits (full attention + mamba "align") -----
def test_coordinator_fine_grained_partial_tail_hit():
"""K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When
both groups have the sub-block boundary hash, the reconciled hit lands on
the hash boundary (48), not the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
assert coord.enable_partial_hash_hits
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
exists = {(g, bytes(h)) for g in (0, 1) for h in (hs[1], hs[2])}
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 48
def test_coordinator_fine_grained_clips_when_one_group_missing_tail():
"""If only one group has the sub-block boundary, min-convergence clips the
reconciled hit back to the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
# Full block 0 for both; partial boundary hs[2] only for FA (group 0).
exists = {(g, bytes(hs[1])) for g in (0, 1)}
exists |= {(0, bytes(hs[2]))}
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 32
# ----- store_mask -----
@@ -32,6 +32,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
SlidingWindowSpec,
)
@@ -333,14 +334,14 @@ def test_recv_skips_swa_blocks_before_window():
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
keys each 16-token chunk by its last fine hash, keeping the Mooncake key
at one digest instead of concatenating all 4 fine hashes."""
keys each chunk by its ending fine hash, including a partial tail."""
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
db.set_kv_caches_base_addr([0])
db.set_block_len([512])
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
assert len(out) == 2
assert out[0][0] == 0 and out[0][1] == 16
@@ -349,3 +350,243 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
# prior three.
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[7].hex()
# Sub-block hit: emit the partial chunk under its ending fine hash.
out = list(db.process_tokens(token_len=12, block_hashes=fine_hashes[:3]))
assert [(s, e) for s, e, _ in out] == [(0, 12)]
assert out[0][2].hex() == fine_hashes[2].hex()
# Cross-block hit: emit both the full chunk and its partial tail.
out = list(db.process_tokens(token_len=28, block_hashes=fine_hashes[:7]))
assert [(s, e) for s, e, _ in out] == [(0, 16), (16, 28)]
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[6].hex()
def test_sub_block_partial_tail_offload_reads_cow_block():
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
partial tail is offloaded for both groups under the boundary sub-hash. The
full-attention block is read from the request block table; the mamba block
is the core-provided CoW target, not block_ids."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
def __init__(self):
super().__init__()
self.puts: dict[str, list[int]] = {}
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
for key, addr in zip(keys, addrs):
self.puts[key] = addr
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _RecordingStore()
token_dbs = []
for g_idx in range(2):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=16,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
# The surrounding metadata may describe a longer resumed replay, but the
# handoff identifies the exact state boundary to persist.
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
mamba_cow_block = 7
req = ReqMeta(
req_id="r0",
token_len_chunk=0,
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
partial_hash = hs[2]
fa_key = token_dbs[0].key_for(partial_hash)
mamba_key = token_dbs[1].key_for(partial_hash)
assert set(store.puts) == {fa_key, mamba_key}
# FA reads block_ids[0][0] = block 1: addr = base(0) + 1 * 512.
assert store.puts[fa_key] == [512]
# Mamba reads the CoW block 7, not block_ids[1][0]=2.
assert store.puts[mamba_key] == [10_000 + mamba_cow_block * 512]
def test_offload_syncs_event_before_put():
"""An offload-carrying meta synchronizes its CoW-fence event before the
store put reads the blocks, then completes in one pass and drains the
completion counter."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
event = MagicMock()
class _FencedStore(_DictStore):
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
assert event.synchronize.called, "put must run after the event sync"
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _FencedStore()
token_dbs = []
for g_idx in range(2):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=16,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)]
req = ReqMeta(
req_id="r1",
token_len_chunk=0,
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, 7, 12)],
)
req.current_event = event
send.add_stored_request("r1")
send.request_queue.put(req)
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
assert send.stored_requests["r1"] == 0
event.synchronize.assert_called_once()
def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
"""The K3-shaped 900/128/1536 scenario scaled to 12/4/16, with a
full-attention group whose block (4) is smaller than the lcm (16): the
offload must persist every FA block up to the boundary — the normal save
floors to the lcm, so those blocks are otherwise never written and the
consumer's per-group lookup would miss. The mamba boundary block still
reads the core-provided CoW target."""
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
def __init__(self):
super().__init__()
self.puts: dict[str, list[int]] = {}
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
for key, addr in zip(keys, addrs):
self.puts[key] = addr
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _RecordingStore()
token_dbs = []
for g_idx, block_size in enumerate([4, 16]):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=block_size,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)] # 3 hash units = 12 tok
mamba_cow_block = 7
req = ReqMeta(
req_id="r2",
token_len_chunk=0,
block_ids=([1, 2, 3], [4]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
# block-end hashes; mamba (block 16): the partial boundary block under
# the boundary sub-hash, read from the CoW target.
expected = {
token_dbs[0].key_for(hs[0]): [1 * 512],
token_dbs[0].key_for(hs[1]): [2 * 512],
token_dbs[0].key_for(hs[2]): [3 * 512],
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
}
assert store.puts == expected
@@ -13,11 +13,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
)
def _make_bare_scheduler() -> MooncakeStoreScheduler:
def _make_bare_scheduler(
*, hash_block_size: int = 16, enable_partial_hash_hits: bool = False
) -> MooncakeStoreScheduler:
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.lookup_async = False
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
scheduler.load_specs = {}
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
@@ -136,6 +140,7 @@ def test_preemption_resets_tracker_before_request_finished():
block_hashes=[b"h0", b"h1"],
prefill_end_tokens=48,
)
scheduler._request_trackers["req-0"].has_pending_offload = True
scheduler.build_connector_meta(_make_preemption_scheduler_output())
@@ -144,6 +149,7 @@ def test_preemption_resets_tracker_before_request_finished():
assert tracker.allocated_block_ids == ()
assert tracker.num_saved_tokens == 0
assert tracker.token_ids is None
assert tracker.has_pending_offload is False
assert tracker.prefill_end_tokens == 0
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
@@ -536,3 +542,227 @@ def test_full_external_hit_with_full_local_hit_skips_load():
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_partial_hash_hit_block_aligned_local_loads_partial_tail():
# Fine-grained on (hash=4, block=16): a block-aligned local hit can pull a
# sub-block remote hit (24 = a hash boundary inside block 1). Loads [16, 24).
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=24)
request = SimpleNamespace(
request_id="req-0",
num_tokens=32,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=16
)
assert need_to_allocate == 8
assert load_async is True
load_spec = scheduler.load_specs["req-0"]
assert load_spec.vllm_cached_tokens == 16
assert load_spec.kvpool_cached_tokens == 24
def test_partial_hash_hit_no_remote_gain_skips_load():
# Core always presents a block-aligned local hit (it floors a sub-block
# tail before calling the connector). When the remote hit does not exceed
# that block-aligned local hit, nothing is loaded.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=16)
request = SimpleNamespace(
request_id="req-0",
num_tokens=32,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=16
)
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_sub_block_prompt_looks_up_with_fine_grained():
# A prompt smaller than one block (12 < block 16). With fine-grained partial
# hits the sub-block prefix is worth looking up (floor is the hash unit 4,
# not a full block), so a remote partial hit is loaded. Pre-change the
# block-size floor returned (0, False) for such prompts.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=8)
request = SimpleNamespace(
request_id="req-0",
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 8
assert load_async is True
assert scheduler.load_specs["req-0"].kvpool_cached_tokens == 8
def test_sub_block_prompt_not_looked_up_without_fine_grained():
# Without fine-grained partial hits, sub-block prompts still skip the lookup
# (there is no full block, and no sub-block key granularity).
scheduler = _make_bare_scheduler()
scheduler.client = _StubLookupClient(hit_tokens=8)
request = SimpleNamespace(
request_id="req-0",
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save). Pending-offload state delays the free
# without advancing the normal-save watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(12)),
block_hashes=[b"h0", b"h1", b"h2"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([0],),
num_saved_tokens=0,
token_ids=list(range(12)),
prefill_end_tokens=12,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 0
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
assert req_meta.num_prompt_tokens == 12
assert req_meta.block_ids == ([0],)
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0],)) == (True, None)
def test_resumed_partial_tail_uses_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(20)),
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=20,
allocated_block_ids=([0, 1],),
num_saved_tokens=0,
token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
prefill_end_tokens=20,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
# Ordinary metadata retains the full resumed prefill range.
assert meta.requests[0].num_prompt_tokens == 20
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_output_placeholders=0,
num_prompt_tokens=36,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=44,
allocated_block_ids=([0, 1],),
num_saved_tokens=32,
token_ids=list(range(44)),
prefill_end_tokens=48,
)
out = _make_scheduler_output(scheduled_spec_tokens=None)
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].can_save is True
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
assert meta.requests[0].num_prompt_tokens == 48
# Ordinary saving still covers the full resumed prefill range.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
assert tracker.has_pending_offload is True
@@ -599,6 +599,88 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def _make_partial_tail_send_thread(store):
coord = SimpleNamespace(
enable_partial_hash_hits=True,
hash_block_size=4,
lcm_block_size=16,
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=4,
hash_block_size=4,
)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([256])
return _make_store_sending_thread(
store,
coord=coord,
token_databases=[db],
)
def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
return ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=(block_ids,),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
partial_tail_offloads=[(1, 7, 12)],
)
def test_partial_tail_offload_skips_null_source_blocks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_partial_tail_send_thread(store)
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
)
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
def test_partial_tail_offload_honors_active_pressure_gate():
store = MagicMock()
thread = _make_partial_tail_send_thread(store)
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["req-a"] == 0
def test_partial_tail_put_failure_activates_pressure_gate():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
thread = _make_partial_tail_send_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
assert thread._store_pressure_active is True
assert thread._skip_store_requests == {"req-a"}
assert thread._saved_offset.get("req-a", 0) == 0
assert thread.stored_requests["req-a"] == 0
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
assert store.batch_put_from_multi_buffers.call_count == 1
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -1939,6 +2021,46 @@ def test_lookup_partial_prefix_returns_first_hit_length():
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_partial_tail_uses_hash_alignment():
"""A stored sub-block tail can serve a request extending past it."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
)
worker = _make_bare_worker(block_size=16)
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
worker._kv_cache_groups = [
KVCacheGroupSpec(["full"], full),
KVCacheGroupSpec(["mamba"], mamba),
]
worker.hash_block_size = 4
worker.token_dbs = [
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
block_size=16,
hash_block_size=4,
)
for group_id in range(2)
]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=4,
)
_refresh_group_tp_replication_factors(worker)
worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1]
assert worker.lookup(13, [b"h0", b"h1", b"h2"]) == 12
def test_lookup_full_hit_reuses_existing_boundary():
"""A full hit is re-derived below the request end without another RPC."""
worker = _make_bare_worker(block_size=16)