[3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Dao007forever
2026-07-26 23:22:17 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 544cb724c8
commit d742856610
13 changed files with 1448 additions and 49 deletions
@@ -223,6 +223,299 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
assert moved[0].block_hash_num_tokens == 6
def test_take_partial_tail_offloads_returns_cow_target():
"""The connector offload hand-off exposes the mamba CoW *target* block Y
(the durable boundary state), not the overwritten source X, and only at
the CoW step."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A registered the partial tail but has not CoW'd yet: no offload.
assert manager.take_partial_tail_offloads() == {}
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert source_block is not None
source_block_id = source_block[0].block_id
# Step B: the producer continues, triggering the CoW X->Y.
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
offloads = manager.take_partial_tail_offloads()
assert list(offloads.keys()) == ["0"]
assert len(offloads["0"]) == 1
group_id, block_id, boundary_tokens = offloads["0"][0]
assert group_id == 1 # the mamba group
assert boundary_tokens == 6
copies, _ = manager.take_kv_cache_block_copies()
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
# The offload points at the durable CoW target Y, not the overwritten X.
assert block_id == cow_copy.dst_block_id
assert block_id != source_block_id
# Draining clears it.
assert manager.take_partial_tail_offloads() == {}
# The hand-off pinned Y (its CoW retention is released after this step,
# and Y is off the request block table); freeing the request unpins it.
cow_block = manager.block_pool.blocks[block_id]
pinned_ref = cow_block.ref_cnt
assert pinned_ref >= 1
manager.free(req0)
assert cow_block.ref_cnt == pinned_ref - 1
def test_partial_tail_pin_survives_released_cow_retention():
"""If the CoW retention is released before the hand-off is drained
(immediate-free mode), the drain must rescue the cow block from the free
queue: a raw ref increment would leave a ref>0 block allocatable, and the
next allocation would pop it and assert."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# Retention released before the drain (defer_block_free=False ordering).
_copies, retained = manager.take_kv_cache_block_copies()
manager.block_pool.free_blocks(retained)
offloads = manager.take_partial_tail_offloads()
((_group_id, block_id, boundary_tokens),) = offloads["0"]
assert boundary_tokens == 6
cow_block = manager.block_pool.blocks[block_id]
assert cow_block.ref_cnt == 1
# The pinned block is out of the free queue: draining every free block
# neither trips the allocator's ref_cnt assert nor hands it out.
new_blocks = manager.block_pool.get_new_blocks(
manager.block_pool.get_num_free_blocks()
)
assert block_id not in {b.block_id for b in new_blocks}
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# The request dies (preempt/abort) before the scheduler drains.
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
assert manager.take_partial_tail_offloads() == {}
def test_take_partial_tail_offloads_empty_without_partial_tail():
"""A prompt ending on a block boundary registers no partial tail, so there
is nothing to offload."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
# 4-token prompt ends exactly on the mamba block boundary (block_size=4).
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
assert manager.take_partial_tail_offloads() == {}
req0.num_computed_tokens = 4
req0.append_output_token_ids([2])
assert manager.allocate_slots(req0, 1) is not None
assert manager.take_partial_tail_offloads() == {}
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
"""truncate_computed_blocks slices each group by its own block size,
keeps null placeholders in the retained prefix, and leaves the original
lookup result untouched (pure view, no refcount changes)."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=2 * hash_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
manager.new_step_starts()
consumer = make_request(
"consumer", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256
)
blocks, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
assert [len(group) for group in blocks.blocks] == [3, 2]
assert blocks.blocks[1][0].is_null
truncated = manager.truncate_computed_blocks(blocks, 4)
assert [len(group) for group in truncated.blocks] == [2, 1]
assert truncated.blocks[1][0].is_null
assert [len(group) for group in blocks.blocks] == [3, 2]
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size = 2
block_size = 2 * hash_block_size
@@ -3,6 +3,8 @@
from math import lcm
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
MooncakeStoreCoordinator,
@@ -14,10 +16,20 @@ from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
)
def _mamba_align(block_size=32):
return MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
@@ -196,6 +208,49 @@ def test_coordinator_group_block_size_double_hash():
assert hit % 32 == 0
# ----- Fine-grained partial hits (full attention + mamba "align") -----
def test_coordinator_fine_grained_partial_tail_hit():
"""K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When
both groups have the sub-block boundary hash, the reconciled hit lands on
the hash boundary (48), not the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
assert coord.enable_partial_hash_hits
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
exists = {(g, bytes(h)) for g in (0, 1) for h in (hs[1], hs[2])}
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 48
def test_coordinator_fine_grained_clips_when_one_group_missing_tail():
"""If only one group has the sub-block boundary, min-convergence clips the
reconciled hit back to the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
# Full block 0 for both; partial boundary hs[2] only for FA (group 0).
exists = {(g, bytes(hs[1])) for g in (0, 1)}
exists |= {(0, bytes(hs[2]))}
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
assert hit == 32
# ----- store_mask -----
@@ -32,6 +32,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
SlidingWindowSpec,
)
@@ -333,14 +334,14 @@ def test_recv_skips_swa_blocks_before_window():
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
keys each 16-token chunk by its last fine hash, keeping the Mooncake key
at one digest instead of concatenating all 4 fine hashes."""
keys each chunk by its ending fine hash, including a partial tail."""
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
db.set_kv_caches_base_addr([0])
db.set_block_len([512])
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
assert len(out) == 2
assert out[0][0] == 0 and out[0][1] == 16
@@ -349,3 +350,243 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
# prior three.
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[7].hex()
# Sub-block hit: emit the partial chunk under its ending fine hash.
out = list(db.process_tokens(token_len=12, block_hashes=fine_hashes[:3]))
assert [(s, e) for s, e, _ in out] == [(0, 12)]
assert out[0][2].hex() == fine_hashes[2].hex()
# Cross-block hit: emit both the full chunk and its partial tail.
out = list(db.process_tokens(token_len=28, block_hashes=fine_hashes[:7]))
assert [(s, e) for s, e, _ in out] == [(0, 16), (16, 28)]
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[6].hex()
def test_sub_block_partial_tail_offload_reads_cow_block():
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
partial tail is offloaded for both groups under the boundary sub-hash. The
full-attention block is read from the request block table; the mamba block
is the core-provided CoW target, not block_ids."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
def __init__(self):
super().__init__()
self.puts: dict[str, list[int]] = {}
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
for key, addr in zip(keys, addrs):
self.puts[key] = addr
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _RecordingStore()
token_dbs = []
for g_idx in range(2):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=16,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
# The surrounding metadata may describe a longer resumed replay, but the
# handoff identifies the exact state boundary to persist.
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
mamba_cow_block = 7
req = ReqMeta(
req_id="r0",
token_len_chunk=0,
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
partial_hash = hs[2]
fa_key = token_dbs[0].key_for(partial_hash)
mamba_key = token_dbs[1].key_for(partial_hash)
assert set(store.puts) == {fa_key, mamba_key}
# FA reads block_ids[0][0] = block 1: addr = base(0) + 1 * 512.
assert store.puts[fa_key] == [512]
# Mamba reads the CoW block 7, not block_ids[1][0]=2.
assert store.puts[mamba_key] == [10_000 + mamba_cow_block * 512]
def test_offload_syncs_event_before_put():
"""An offload-carrying meta synchronizes its CoW-fence event before the
store put reads the blocks, then completes in one pass and drains the
completion counter."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
event = MagicMock()
class _FencedStore(_DictStore):
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
assert event.synchronize.called, "put must run after the event sync"
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _FencedStore()
token_dbs = []
for g_idx in range(2):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=16,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)]
req = ReqMeta(
req_id="r1",
token_len_chunk=0,
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, 7, 12)],
)
req.current_event = event
send.add_stored_request("r1")
send.request_queue.put(req)
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
assert send.stored_requests["r1"] == 0
event.synchronize.assert_called_once()
def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
"""The K3-shaped 900/128/1536 scenario scaled to 12/4/16, with a
full-attention group whose block (4) is smaller than the lcm (16): the
offload must persist every FA block up to the boundary — the normal save
floors to the lcm, so those blocks are otherwise never written and the
consumer's per-group lookup would miss. The mamba boundary block still
reads the core-provided CoW target."""
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
groups = [
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
def __init__(self):
super().__init__()
self.puts: dict[str, list[int]] = {}
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *a, **k):
for key, addr in zip(keys, addrs):
self.puts[key] = addr
return super().batch_put_from_multi_buffers(keys, addrs, sizes, *a, **k)
store = _RecordingStore()
token_dbs = []
for g_idx, block_size in enumerate([4, 16]):
db = ChunkedTokenDatabase(
KeyMetadata("m", 0, 0, 0, 0, group_id=g_idx),
block_size=block_size,
hash_block_size=4,
)
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
token_dbs.append(db)
send = KVCacheStoreSendingThread(
store=store,
coord=coord,
token_databases=token_dbs,
block_size=16,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(3)] # 3 hash units = 12 tok
mamba_cow_block = 7
req = ReqMeta(
req_id="r2",
token_len_chunk=0,
block_ids=([1, 2, 3], [4]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
# block-end hashes; mamba (block 16): the partial boundary block under
# the boundary sub-hash, read from the CoW target.
expected = {
token_dbs[0].key_for(hs[0]): [1 * 512],
token_dbs[0].key_for(hs[1]): [2 * 512],
token_dbs[0].key_for(hs[2]): [3 * 512],
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
}
assert store.puts == expected
@@ -13,11 +13,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
)
def _make_bare_scheduler() -> MooncakeStoreScheduler:
def _make_bare_scheduler(
*, hash_block_size: int = 16, enable_partial_hash_hits: bool = False
) -> MooncakeStoreScheduler:
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.lookup_async = False
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
scheduler.load_specs = {}
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
@@ -136,6 +140,7 @@ def test_preemption_resets_tracker_before_request_finished():
block_hashes=[b"h0", b"h1"],
prefill_end_tokens=48,
)
scheduler._request_trackers["req-0"].has_pending_offload = True
scheduler.build_connector_meta(_make_preemption_scheduler_output())
@@ -144,6 +149,7 @@ def test_preemption_resets_tracker_before_request_finished():
assert tracker.allocated_block_ids == ()
assert tracker.num_saved_tokens == 0
assert tracker.token_ids is None
assert tracker.has_pending_offload is False
assert tracker.prefill_end_tokens == 0
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
@@ -536,3 +542,227 @@ def test_full_external_hit_with_full_local_hit_skips_load():
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_partial_hash_hit_block_aligned_local_loads_partial_tail():
# Fine-grained on (hash=4, block=16): a block-aligned local hit can pull a
# sub-block remote hit (24 = a hash boundary inside block 1). Loads [16, 24).
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=24)
request = SimpleNamespace(
request_id="req-0",
num_tokens=32,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=16
)
assert need_to_allocate == 8
assert load_async is True
load_spec = scheduler.load_specs["req-0"]
assert load_spec.vllm_cached_tokens == 16
assert load_spec.kvpool_cached_tokens == 24
def test_partial_hash_hit_no_remote_gain_skips_load():
# Core always presents a block-aligned local hit (it floors a sub-block
# tail before calling the connector). When the remote hit does not exceed
# that block-aligned local hit, nothing is loaded.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=16)
request = SimpleNamespace(
request_id="req-0",
num_tokens=32,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5", b"h6", b"h7"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=16
)
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_sub_block_prompt_looks_up_with_fine_grained():
# A prompt smaller than one block (12 < block 16). With fine-grained partial
# hits the sub-block prefix is worth looking up (floor is the hash unit 4,
# not a full block), so a remote partial hit is loaded. Pre-change the
# block-size floor returned (0, False) for such prompts.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=8)
request = SimpleNamespace(
request_id="req-0",
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 8
assert load_async is True
assert scheduler.load_specs["req-0"].kvpool_cached_tokens == 8
def test_sub_block_prompt_not_looked_up_without_fine_grained():
# Without fine-grained partial hits, sub-block prompts still skip the lookup
# (there is no full block, and no sub-block key granularity).
scheduler = _make_bare_scheduler()
scheduler.client = _StubLookupClient(hit_tokens=8)
request = SimpleNamespace(
request_id="req-0",
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 0
assert load_async is False
assert "req-0" not in scheduler.load_specs
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save). Pending-offload state delays the free
# without advancing the normal-save watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(12)),
block_hashes=[b"h0", b"h1", b"h2"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([0],),
num_saved_tokens=0,
token_ids=list(range(12)),
prefill_end_tokens=12,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 0
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
assert req_meta.num_prompt_tokens == 12
assert req_meta.block_ids == ([0],)
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0],)) == (True, None)
def test_resumed_partial_tail_uses_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(20)),
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=20,
allocated_block_ids=([0, 1],),
num_saved_tokens=0,
token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
prefill_end_tokens=20,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
# Ordinary metadata retains the full resumed prefill range.
assert meta.requests[0].num_prompt_tokens == 20
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_output_placeholders=0,
num_prompt_tokens=36,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=44,
allocated_block_ids=([0, 1],),
num_saved_tokens=32,
token_ids=list(range(44)),
prefill_end_tokens=48,
)
out = _make_scheduler_output(scheduled_spec_tokens=None)
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].can_save is True
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
assert meta.requests[0].num_prompt_tokens == 48
# Ordinary saving still covers the full resumed prefill range.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
assert tracker.has_pending_offload is True
@@ -599,6 +599,88 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def _make_partial_tail_send_thread(store):
coord = SimpleNamespace(
enable_partial_hash_hits=True,
hash_block_size=4,
lcm_block_size=16,
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=4,
hash_block_size=4,
)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([256])
return _make_store_sending_thread(
store,
coord=coord,
token_databases=[db],
)
def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
return ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=(block_ids,),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
partial_tail_offloads=[(1, 7, 12)],
)
def test_partial_tail_offload_skips_null_source_blocks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_partial_tail_send_thread(store)
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
)
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
def test_partial_tail_offload_honors_active_pressure_gate():
store = MagicMock()
thread = _make_partial_tail_send_thread(store)
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["req-a"] == 0
def test_partial_tail_put_failure_activates_pressure_gate():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
thread = _make_partial_tail_send_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
assert thread._store_pressure_active is True
assert thread._skip_store_requests == {"req-a"}
assert thread._saved_offset.get("req-a", 0) == 0
assert thread.stored_requests["req-a"] == 0
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
assert store.batch_put_from_multi_buffers.call_count == 1
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -1939,6 +2021,46 @@ def test_lookup_partial_prefix_returns_first_hit_length():
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_partial_tail_uses_hash_alignment():
"""A stored sub-block tail can serve a request extending past it."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
)
worker = _make_bare_worker(block_size=16)
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
worker._kv_cache_groups = [
KVCacheGroupSpec(["full"], full),
KVCacheGroupSpec(["mamba"], mamba),
]
worker.hash_block_size = 4
worker.token_dbs = [
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
block_size=16,
hash_block_size=4,
)
for group_id in range(2)
]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=4,
)
_refresh_group_tp_replication_factors(worker)
worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1]
assert worker.lookup(13, [b"h0", b"h1", b"h2"]) == 12
def test_lookup_full_hit_reuses_existing_boundary():
"""A full hit is re-derived below the request end without another RPC."""
worker = _make_bare_worker(block_size=16)
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
@@ -83,6 +84,9 @@ class MooncakeStoreCoordinator:
self.kv_cache_groups = kv_cache_groups
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_groups, hash_block_size
)
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
@@ -94,7 +98,12 @@ class MooncakeStoreCoordinator:
self._verify_and_split_kv_cache_groups()
def align_lookup_length(self, length: int) -> int:
return length // self.lcm_block_size * self.lcm_block_size
alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
return length // alignment * alignment
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
@@ -226,9 +235,14 @@ class MooncakeStoreCoordinator:
retention_interval: int | None,
num_prompt_tokens: int | None,
) -> tuple[list[bool] | None, ...]:
assert aligned_token_len % self.lcm_block_size == 0, (
mask_alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
assert aligned_token_len % mask_alignment == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"lcm_block_size ({self.lcm_block_size})"
f"{mask_alignment}"
)
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
@@ -278,17 +292,21 @@ class MooncakeStoreCoordinator:
one already removed by the lookup.
"""
eagle_indices = self.eagle_attn_group_indices if apply_eagle else set()
alignment_tokens = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls = self.attention_groups[0]
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=(0 in eagle_indices),
alignment_tokens=spec.block_size,
alignment_tokens=alignment_tokens,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
@@ -320,17 +338,23 @@ class MooncakeStoreCoordinator:
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
_max_length = curr_hit_length
if drop_eagle_block:
_max_length = min(curr_hit_length + spec.block_size, max_length)
hashes = self.block_hashes_for_spec(block_hashes, spec)
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = (
self.hash_block_size
if self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup
and spec.block_size > self.hash_block_size
else spec.block_size
)
_max_length = min(curr_hit_length + eagle_margin, max_length)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=self.lcm_block_size,
alignment_tokens=alignment_tokens,
)
if drop_eagle_block:
eagle_verified.add(idx)
@@ -348,10 +372,11 @@ class MooncakeStoreCoordinator:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic.
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
spec0, group_ids0, _ = self.attention_groups[0]
if isinstance(spec0, FullAttentionSpec):
num_blocks = hit_length // spec0.block_size
num_blocks = cdiv(hit_length, spec0.block_size)
for gid in group_ids0:
full_blks = hit_blocks_by_group[gid]
assert full_blks is not None
@@ -368,3 +393,18 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
def partial_hash_hits_enabled(
kv_cache_groups: list[KVCacheGroupSpec], hash_block_size: int
) -> bool:
"""Mirror of core's ``HybridKVCacheCoordinator.enable_partial_hash_hits``
(its dcp == 1 clause holds: the connector rejects hybrid + DCP/PCP > 1).
Single copy on purpose — scheduler and coordinator must not disagree.
"""
return any(
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
and spec.mamba_cache_mode == "align"
and spec.block_size > hash_block_size
for g in kv_cache_groups
)
@@ -228,16 +228,28 @@ class ChunkedTokenDatabase:
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert not (spans % self.block_size).any()
assert not (spans % self.hash_block_size).any()
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
sizes = blen[None, :] * (spans // self.block_size)[:, None]
block_counts = (spans + self.block_size - 1) // self.block_size
sizes = blen[None, :] * block_counts[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
"""Return addresses and sizes for one physical block slot."""
addr_list = []
size_list = []
length = len(self.block_len)
for index, base_addr in enumerate(self.kv_caches_base_addr):
addr = base_addr + block_id * self.block_len[index % length]
addr_list.append(addr)
size_list.append(self.block_len[index % length])
return addr_list, size_list
def process_tokens(
self,
token_len: int,
@@ -256,7 +268,8 @@ class ChunkedTokenDatabase:
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens.
token_len: Total number of tokens. Must be hash-block aligned and
covered by ``block_hashes`` when hashes are present.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
@@ -269,11 +282,10 @@ class ChunkedTokenDatabase:
assert put_step > 0
if not block_hashes:
return
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, self.block_size
)
assert token_len % self.hash_block_size == 0
assert token_len // self.hash_block_size <= len(block_hashes)
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
max_chunks = cdiv(token_len, self.block_size)
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
@@ -281,9 +293,9 @@ class ChunkedTokenDatabase:
continue
if chunk_id % put_step != put_step_rank:
continue
h = chunk_hashes[chunk_id]
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
h = block_hashes[end_idx // self.hash_block_size - 1]
yield start_idx, end_idx, h
@@ -306,6 +318,7 @@ class RequestTracker:
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
has_pending_offload: bool = False
# Snapshot of the prefill range length at tracker creation time.
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
@@ -316,6 +329,7 @@ class RequestTracker:
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
def update(
@@ -352,6 +366,11 @@ class ReqMeta:
token_ids: list[int] | None = None
num_prompt_tokens: int | None = None
# Core-provided per-mamba-group
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
# Present only on the producer's CoW step; drives the connector's offload
# (the FA group's block is derived from block_ids and boundary_tokens).
partial_tail_offloads: list[tuple[int, int, int]] | None = None
@staticmethod
def from_request_tracker(
@@ -11,6 +11,9 @@ from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
partial_hash_hits_enabled,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
LoadSpec,
MooncakeStoreConnectorMetadata,
@@ -63,6 +66,9 @@ class MooncakeStoreScheduler:
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_config.kv_cache_groups, self._hash_block_size
)
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
@@ -80,7 +86,12 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if request.num_tokens < self._block_size:
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
)
if request.num_tokens < align:
return 0, False
num_external_hit_tokens = self.client.lookup(
@@ -338,6 +349,44 @@ class MooncakeStoreScheduler:
if req_meta is not None:
meta.add_request(req_meta)
# Flush partial-tail offloads in the step they arrive: the CoW copy is
# enqueued before the connector event records, so this step's event
# fences the cow block. Ride the request's save meta when present, else
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
# save; can_save=True takes the normal enqueue path).
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
if step_partial_tails and not force_skip_save:
pending = dict(step_partial_tails)
for req_meta in meta.requests:
if req_meta.can_save:
groups = pending.pop(req_meta.req_id, None)
if groups:
req_meta.partial_tail_offloads = groups
tracker = self._request_trackers.get(req_meta.req_id)
if tracker is not None:
tracker.has_pending_offload = True
for req_id, groups in pending.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks
# are going away, so the offload is conservatively dropped.
logger.debug("Dropping partial-tail offload for request %s", req_id)
continue
assert len({boundary for _, _, boundary in groups}) == 1
tracker.has_pending_offload = True
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
partial_tail_offloads=groups,
)
)
return meta
def request_finished(
@@ -352,7 +401,9 @@ class MooncakeStoreScheduler:
# Missing tracker can happen when the request is aborted before the
# connector observes the normal finished lifecycle or is preempted
# before finishing.
if tracker is None or tracker.num_saved_tokens <= 0:
if tracker is None or (
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
):
return False, None
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
@@ -61,6 +61,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
@@ -531,6 +532,164 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.clear()
return True
def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool:
"""Offload the request's sub-block partial tail (its last prompt hash
boundary) so a later request can hit the sub-block prefix.
Covers every block from the normal save's lcm floor to the boundary:
the normal save floors to ``lcm_block_size``, so a smaller-block
group's full blocks in that gap are never persisted elsewhere, and
the consumer's lookup needs every group at every probed boundary.
Full blocks are keyed by their block-end hash, the partial boundary
block by the boundary sub-hash; the mamba "align" boundary block is
the core-provided CoW block. All keys are deduped against the store.
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes:
return True
partial_tail_offloads = req_meta.partial_tail_offloads
if not partial_tail_offloads:
return True
hash_block_size = self.coord.hash_block_size
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary = boundaries.pop()
if boundary == 0:
return True
if boundary // hash_block_size - 1 >= len(req_meta.block_hashes):
return True
mamba_offloads = {
group_id: block_id for group_id, block_id, _ in partial_tail_offloads
}
keys: list[str] = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
saved = self._saved_offset.get(req_meta.req_id, 0)
for g_idx, db in enumerate(self.token_databases):
group_blocks = req_meta.block_ids[g_idx]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[g_idx]
put_step_rank = (self.tp_rank + g_idx) % put_step
# Always include the boundary block: its sub-hash key is written
# only here, even if normal saves already advanced past it.
last_block = cdiv(boundary, db.block_size) - 1
for block_idx in range(
min(saved // db.block_size, last_block), last_block + 1
):
if block_idx % put_step != put_step_rank:
continue
valid_end = min((block_idx + 1) * db.block_size, boundary)
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
if (
g_idx in mamba_offloads
and valid_end == boundary
and boundary % db.block_size != 0
):
block_id = mamba_offloads[g_idx]
else:
if block_idx >= len(group_blocks):
continue
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
logger.debug(
"Skipping unavailable partial-tail source block "
"(req=%s, group=%d, block=%d)",
req_meta.req_id,
g_idx,
block_idx,
)
continue
addr, size = db.prepare_value_for_block(block_id)
keys.append(db.key_for(key_hash))
addrs.append(addr)
sizes.append(size)
if not keys:
return True
exists_start = time.perf_counter()
try:
exists = self.store.batch_is_exist(keys)
except Exception as e:
self._record_operation(
"save_exists",
exists_start,
len(keys),
status="error",
num_failed_keys=len(keys),
)
logger.error(
"Failed to check partial-tail keys for request %s: %s",
req_meta.req_id,
e,
)
return False
self._record_operation("save_exists", exists_start, len(keys))
missing = [i for i, e in enumerate(exists) if e != 1]
if not missing:
return True
keys = [keys[i] for i in missing]
addrs = [addrs[i] for i in missing]
sizes = [sizes[i] for i in missing]
if req_meta.current_event is not None:
# Fence the CoW block copy enqueued earlier this step.
req_meta.current_event.synchronize()
batch_bytes = _sum_batch_bytes(sizes)
put_start = time.perf_counter()
try:
res = self.store.batch_put_from_multi_buffers(
keys, addrs, sizes, self.replicate_config
)
except Exception as e:
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="error",
num_failed_keys=len(keys),
)
logger.error(
"Failed to put partial-tail keys for request %s: %s",
req_meta.req_id,
e,
)
return False
failed = [i for i, value in enumerate(res) if value < 0]
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="partial_failure" if failed else "ok",
num_failed_keys=len(failed),
)
if failed:
failed_codes = {res[i] for i in failed}
logger.warning(
"Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)",
req_meta.req_id,
len(failed),
len(keys),
failed_codes,
)
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
self._mark_request_skipped_for_pressure(req_meta.req_id)
return False
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared after a "
"successful partial-tail batch"
)
return True
def _handle_request(self, req_meta: ReqMeta):
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
# is also ``store_mask``'s precondition.
@@ -548,9 +707,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
# so the scheduler can release the GPU blocks it pinned for this
# request (via `delay_free_blocks`) even when the store path raises.
try:
if token_len == 0:
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -559,6 +715,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
return
# Offload the sub-block partial tail (independent of the normal
# block-aligned save, which may be skipped this step).
if req_meta.partial_tail_offloads is not None and not (
self._maybe_offload_partial_tail(req_meta)
):
return
if token_len == 0:
return
# Resume from where this rank left off; only the new suffix is saved.
save_start = self._saved_offset.get(req_id, 0)
@@ -1385,7 +1551,7 @@ class MooncakeStoreWorker:
self.recv_request_queue.put(request)
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization
# Issue stores with CUDA event synchronization.
if self.kv_role in ["kv_producer", "kv_both"]:
current_event = None
for request in meta.requests:
@@ -1501,21 +1667,32 @@ class MooncakeStoreWorker:
# candidate_meta stores the (group, hash_bytes) for key slice.
candidate_keys: list[str] = []
candidate_meta: list[tuple[int, bytes]] = []
lookup_masks = self.coord.lookup_mask(token_len)
fine_grained = self.coord.enable_partial_hash_hits
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
for g_idx, db in enumerate(self.token_dbs):
spec_block_size = db.block_size
lookup_mask = lookup_masks[g_idx]
key_prefixes = self._lookup_key_prefixes[g_idx]
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
mask_limit = (
max_chunks if lookup_mask is None else min(max_chunks, len(lookup_mask))
)
for chunk_id in range(mask_limit):
if lookup_mask is not None and not lookup_mask[chunk_id]:
continue
if fine_grained:
max_units = min(len(block_hashes), token_len // self.hash_block_size)
unit_ids: range | list[int] = range(max_units)
group_hashes: Sequence[BlockHash] = block_hashes
else:
lookup_mask = lookup_masks[g_idx] # type: ignore[index]
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
mask_limit = (
max_chunks
if lookup_mask is None
else min(max_chunks, len(lookup_mask))
)
unit_ids = [
chunk_id
for chunk_id in range(mask_limit)
if lookup_mask is None or lookup_mask[chunk_id]
]
for chunk_id in unit_ids:
h = group_hashes[chunk_id]
hash_hex = h.hex()
for key_prefix in key_prefixes:
+62 -1
View File
@@ -186,6 +186,10 @@ class KVCacheManager:
tuple(() for _ in range(self.num_kv_cache_groups))
)
# Off-table cow blocks handed to a KV connector for partial-tail
# offload; pinned until the request's blocks are freed.
self._partial_tail_pins: dict[str, list[KVCacheBlock]] = {}
@property
def usage(self) -> float:
"""Get the KV cache usage.
@@ -568,6 +572,9 @@ class KVCacheManager:
Args:
request: The request to free the blocks.
"""
pins = self._partial_tail_pins.pop(request.request_id, None)
if pins:
self.block_pool.free_blocks(pins)
self.coordinator.free(request.request_id)
def remove_skipped_blocks(
@@ -600,7 +607,14 @@ class KVCacheManager:
Returns:
The request's blocks in allocation order.
"""
return self.coordinator.pop_blocks_for_free(request.request_id)
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
# Pins ride the same (possibly deferred) free as the request blocks.
# Preemption may release a pin under a still-queued offload — the same
# exposure normal saves of table blocks already have.
pins = self._partial_tail_pins.pop(request.request_id, None)
if pins:
blocks = pins + blocks
return blocks
def evict_blocks(self, block_ids: set[int]) -> None:
"""evict blocks from the prefix cache by their block IDs.
@@ -760,6 +774,25 @@ class KVCacheManager:
# Only create new KVCacheBlocks for non-empty blocks
return KVCacheBlocks(blocks) if any(blocks) else self.empty_kv_cache_blocks
def truncate_computed_blocks(
self, blocks: KVCacheBlocks, num_computed_tokens: int
) -> KVCacheBlocks:
"""Return a lookup-result view truncated at an aligned token endpoint.
Pure slicing: refcounts are untouched and ``blocks`` is not mutated.
"""
truncated: list[list[KVCacheBlock]] = []
for group_blocks, manager in zip(
blocks.blocks,
self.coordinator.single_type_managers,
strict=True,
):
assert num_computed_tokens % manager.block_size == 0
num_blocks = num_computed_tokens // manager.block_size
assert num_blocks <= len(group_blocks)
truncated.append(list(group_blocks[:num_blocks]))
return self.create_kv_cache_blocks(tuple(truncated))
def take_new_block_ids(self) -> list[int]:
"""Drain and return new attention block IDs for zeroing."""
ids: list[int] = []
@@ -812,6 +845,34 @@ class KVCacheManager:
retained_blocks = [block for pair in pending_copies for block in pair]
return copies, retained_blocks
def take_partial_tail_offloads(self) -> dict[str, list[tuple[int, int, int]]]:
"""Drain producer partial-tail offload hand-offs per request.
Returns ``{request_id: [(group_id, block_id, boundary_tokens), ...]}``
for the durable boundary blocks of producers' last-prompt-boundary
partial tails. Only mamba "align" groups contribute; empty otherwise.
A KV connector reads the referenced blocks and offloads them so a later
request can hit the sub-block prefix.
Each handed-off block lives off the request block table, so it is
pinned here and unpinned when the request's blocks are freed — for a
producer with saved tokens, after the connector reports sends done.
"""
offloads: dict[str, list[tuple[int, int, int]]] = {}
for mgr in self.coordinator.single_type_managers:
for (
req_id,
group_id,
block,
boundary_tokens,
) in mgr.take_pending_partial_tail_offloads():
self.block_pool.touch((block,))
self._partial_tail_pins.setdefault(req_id, []).append(block)
offloads.setdefault(req_id, []).append(
(group_id, block.block_id, boundary_tokens)
)
return offloads
def new_step_starts(self) -> None:
"""Notify the coordinator that a new step is starting."""
self.coordinator.new_step_starts()
+6
View File
@@ -258,6 +258,12 @@ class SchedulerOutput:
# CoW copies to apply after zeroing new blocks and before forward.
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
# Producer partial-tail offload hand-off for external KV connectors:
# {request_id: [(group_id, block_id, boundary_tokens), ...]} pointing at
# the durable boundary block of a producer's last-prompt-boundary partial
# tail (mamba "align" CoW target). None unless partial hash hits are active.
partial_tail_offloads: dict[str, list[tuple[int, int, int]]] | None = None
# Dynamic speculative decoding: optimal K chosen by scheduler.
# Number of spec tokens to schedule for the next step.
num_spec_tokens_to_schedule: int = 0
+46 -2
View File
@@ -740,9 +740,16 @@ class Scheduler(SchedulerInterface):
# Get externally-cached tokens if using a KVConnector.
if self.connector is not None:
# Present a block-aligned local hit to the connector so
# a strictly longer remote hit can supersede a local
# sub-block tail without racing its copy-on-write.
partial_tail = num_new_local_computed_tokens % self.block_size
block_aligned_local = (
num_new_local_computed_tokens - partial_tail
)
ext_tokens, load_kv_async = (
self.connector.get_num_new_matched_tokens(
request, num_new_local_computed_tokens
request, block_aligned_local
)
)
@@ -754,7 +761,27 @@ class Scheduler(SchedulerInterface):
step_skipped_waiting.prepend_request(request)
continue
num_external_computed_tokens = ext_tokens
if partial_tail and ext_tokens > partial_tail:
# Remote strictly exceeds the full local hit: drop the
# sub-block tail so no CoW is needed, and let the load
# cover it. Trim the partial block out of the local
# computed blocks so it is not adopted from the cache.
new_computed_blocks = (
self.kv_cache_manager.truncate_computed_blocks(
new_computed_blocks, block_aligned_local
)
)
num_new_local_computed_tokens = block_aligned_local
num_external_computed_tokens = ext_tokens
elif partial_tail:
# Remote does not exceed the full local hit: keep the
# local sub-block tail and load nothing external.
num_external_computed_tokens = 0
# Nothing to load remotely -> not an async-load step;
# clearing avoids the `load_kv_async` assert below.
load_kv_async = False
else:
num_external_computed_tokens = ext_tokens
if hit_diverged and num_external_computed_tokens == 0:
# No external tokens back the deeper local hit, so its
@@ -1108,6 +1135,22 @@ class Scheduler(SchedulerInterface):
self.prev_step_scheduled_req_ids.clear()
self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys())
# Producer partial-tail hand-off for external KV connectors. Drained
# before the CoW retentions are released below, so the pin lands while
# the cow block still holds a retention ref. Without a producer-side
# connector nothing consumes the hand-off, so skip the drain (and its
# pin); the manager drops stale entries when the request's blocks are
# popped for free.
pending_partial_tail_offloads = None
if (
self.connector is not None
and self.vllm_config.kv_transfer_config is not None
and self.vllm_config.kv_transfer_config.is_kv_producer
):
pending_partial_tail_offloads = (
self.kv_cache_manager.take_partial_tail_offloads() or None
)
kv_cache_block_copies, cow_retained_blocks = (
self.kv_cache_manager.take_kv_cache_block_copies()
)
@@ -1153,6 +1196,7 @@ class Scheduler(SchedulerInterface):
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
kv_cache_block_copies=pending_kv_cache_block_copies,
partial_tail_offloads=pending_partial_tail_offloads,
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
)
+63 -3
View File
@@ -115,6 +115,15 @@ class SingleTypeKVCacheManager(ABC):
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = []
# Partial-tail offload hand-off for external KV connectors: when a
# producer registers its last-prompt-boundary partial tail and the
# durable boundary block is not on the append-only request block table
# (mamba "align" CoW target), record the request, group, block, and
# exact token boundary so a connector can offload it under the right
# hash. Populated only by mamba "align".
self._pending_partial_tail_offloads: list[
tuple[str, int, KVCacheBlock, int]
] = []
@classmethod
def _get_num_evictable_blocks(cls, blocks: Sequence[KVCacheBlock]):
@@ -378,6 +387,21 @@ class SingleTypeKVCacheManager(ABC):
self._pending_cow_copies = []
return pending_copies
def take_pending_partial_tail_offloads(
self,
) -> list[tuple[str, int, KVCacheBlock, int]]:
"""Drain producer partial-tail hand-offs.
Entries are ``(req_id, group_id, block, boundary_tokens)``.
Only mamba "align" populates this. The block lives off the request
block table, so the caller must pin it until the connector has read
it — nothing else keeps it alive once the CoW retention is released.
"""
pending = self._pending_partial_tail_offloads
self._pending_partial_tail_offloads = []
return pending
def _apply_cow(
self,
request_id: str,
@@ -693,7 +717,8 @@ class FullAttentionManager(SingleTypeKVCacheManager):
alignment_tokens < block_size and block_size % alignment_tokens == 0
)
if fine_grained:
assert isinstance(block_hashes, list)
# list or lazy BlobBlockHashes view
assert isinstance(block_hashes, Sequence)
full_block_hashes: BlockHashList = BlockHashListWithBlockSize(
block_hashes, alignment_tokens, block_size
)
@@ -716,7 +741,8 @@ class FullAttentionManager(SingleTypeKVCacheManager):
# Phase 2 (fine-grained only): extend into the first non-full block by
# probing its interior hash boundaries high-to-low (longest hit first).
if fine_grained:
assert isinstance(block_hashes, list)
# list or lazy BlobBlockHashes view
assert isinstance(block_hashes, Sequence)
scale_factor = block_size // alignment_tokens
first_partial_idx = len(computed_blocks[0]) * scale_factor
max_partial_idx = min(
@@ -1244,6 +1270,11 @@ class MambaManager(SingleTypeKVCacheManager):
self.last_state_block_idx: dict[str, int] = {}
# The set of the requests that have been allocated blocks
self._allocated_block_reqs: set[str] = set()
# Requests that registered their own last-prompt-boundary partial
# tail (producers). On the next step's CoW the boundary state moves
# into a private cow_block; we record that block for connector
# offload (see _pending_partial_tail_offloads).
self._producer_partial_tail_reqs: dict[str, int] = {}
@classmethod
def find_longest_cache_hit(
@@ -1277,7 +1308,8 @@ class MambaManager(SingleTypeKVCacheManager):
block_size = kv_cache_spec.block_size
if alignment_tokens < block_size and block_size % alignment_tokens == 0:
assert isinstance(block_hashes, list)
# list or lazy BlobBlockHashes view
assert isinstance(block_hashes, Sequence)
hash_block_size = alignment_tokens
scale_factor = block_size // hash_block_size
max_num_partial_units = min(
@@ -1590,6 +1622,21 @@ class MambaManager(SingleTypeKVCacheManager):
self.block_pool.move_block_hashes(source_block, cow_block)
self._pending_cow_copies.append((source_block, cow_block))
source_block.ref_cnt += 1
boundary_tokens = self._producer_partial_tail_reqs.pop(
request_id, None
)
if boundary_tokens is not None:
# This CoW preserved a producer's own boundary
# state in cow_block; hand it to the connector for
# partial-tail offload once the copy has run.
self._pending_partial_tail_offloads.append(
(
request_id,
self.kv_cache_group_id,
cow_block,
boundary_tokens,
)
)
if cow_block.block_hash is not None:
# The moved entry is only filled by this step's
# copy, so defer same-step hits on it.
@@ -1607,6 +1654,14 @@ class MambaManager(SingleTypeKVCacheManager):
if self.mamba_cache_mode == "align":
self._allocated_block_reqs.discard(request_id)
self.last_state_block_idx.pop(request_id, None)
self._producer_partial_tail_reqs.pop(request_id, None)
# A hand-off whose request died in this same scheduling pass must
# not reach the connector: its unpin hook (free) has already run.
self._pending_partial_tail_offloads = [
entry
for entry in self._pending_partial_tail_offloads
if entry[0] != request_id
]
return super().pop_blocks_for_free(request_id)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
@@ -1681,6 +1736,11 @@ class MambaManager(SingleTypeKVCacheManager):
if partial_hash is not None:
self._partial_hit_reqs[request.request_id] = (block_idx, source_block)
self.num_cached_block[request.request_id] = block_idx
# Producer of this partial tail: the boundary state currently lives
# in ``source_block`` but the next step's forward overwrites it. The
# upcoming CoW copies it into a durable cow_block; record the req so
# allocate_new_blocks hands that block to the connector for offload.
self._producer_partial_tail_reqs[request.request_id] = num_tokens
return partial_hash