[Mooncake] Only check and store new KV cache range (#46412)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Wei Zhao
2026-06-24 03:10:50 -07:00
committed by GitHub
co-authored by Yifan Qiao
parent ede54b926e
commit d20dbf921b
6 changed files with 488 additions and 61 deletions
@@ -246,6 +246,20 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
assert masks[1] == [False, False, True, True, False, False, True, True]
def test_store_mask_swa_prefix_stable_as_aligned_length_grows():
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
shorter = coord.store_mask(64)[1]
longer = coord.store_mask(128)[1]
assert shorter is not None
assert longer is not None
assert longer[: len(shorter)] == shorter
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
@@ -355,6 +369,33 @@ def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
def test_store_mask_suffix_matches_full_mask_tail():
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
full = coord.store_mask(128, num_prompt_tokens=100)
suffix = coord.store_mask(128, start_token=64, num_prompt_tokens=100)
for g_idx, cache_group in enumerate(coord.kv_cache_groups):
block_size = cache_group.kv_cache_spec.block_size
start_chunk = 64 // block_size
end_chunk = 128 // block_size
full_mask = full[g_idx]
if full_mask is None:
assert suffix[g_idx] is None
else:
assert suffix[g_idx] == full_mask[start_chunk:end_chunk]
def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
shorter = coord.store_mask(64, num_prompt_tokens=100)[1]
longer = coord.store_mask(128, num_prompt_tokens=100)[1]
assert shorter is not None
assert longer is not None
assert longer[: len(shorter)] == shorter
# ----- Eagle / MTP interaction with load_mask -----
@@ -337,5 +337,5 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
assert out[1][0] == 16 and out[1][1] == 32
# Each chunk's hash is its last (4th) fine hash, which already chains the
# prior three.
assert out[0][2].chunk_hash == fine_hashes[3].hex()
assert out[1][2].chunk_hash == fine_hashes[7].hex()
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[7].hex()
@@ -36,6 +36,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
from vllm.v1.core.kv_cache_utils import BlockHash
class _RecordingBlockHashes:
def __init__(self, values: list[bytes]):
self.values = values
self.accessed: list[int] = []
def __len__(self):
return len(self.values)
def __getitem__(self, idx):
if isinstance(idx, slice):
return [self[i] for i in range(*idx.indices(len(self)))]
if idx < 0:
idx += len(self.values)
if not 0 <= idx < len(self.values):
raise IndexError(idx)
self.accessed.append(idx)
return self.values[idx]
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
@@ -53,6 +72,8 @@ def _make_store_sending_thread(
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
token_databases: list[ChunkedTokenDatabase] | None = None,
block_size: int = 16,
tp_rank: int = 0,
put_step: int = 1,
replicate_config: object | None = None,
) -> mooncake_store_worker.KVCacheStoreSendingThread:
if coord is None:
@@ -67,8 +88,8 @@ def _make_store_sending_thread(
token_databases=token_databases,
block_size=block_size,
coord=coord,
tp_rank=0,
put_step=1,
tp_rank=tp_rank,
put_step=put_step,
kv_role="kv_producer",
ready_event=threading.Event(),
replicate_config=replicate_config,
@@ -391,6 +412,248 @@ def test_store_sending_thread_records_mooncake_metrics():
assert stats.data["save_put"][0]["status"] == "ok"
def test_process_tokens_uses_mask_num_as_start_chunk():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=96,
block_hashes=block_hashes,
mask_num=64,
)
)
assert block_hashes.accessed == [11]
assert results == [(64, 96, bytes([11]))]
def test_process_tokens_applies_chunk_mask_before_hash_access():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=128,
block_hashes=block_hashes,
mask_num=64,
chunk_mask=[False, True],
)
)
assert block_hashes.accessed == [15]
assert results == [(96, 128, bytes([15]))]
def test_process_tokens_applies_stride_before_hash_access():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=128,
block_hashes=block_hashes,
mask_num=64,
put_step=2,
put_step_rank=1,
)
)
assert block_hashes.accessed == [15]
assert results == [(96, 128, bytes([15]))]
def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_strides_with_local_phase():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256]
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_retries_skipped_range_after_pressure():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
# Under pressure the request is skipped without persisting anything, so the
# saved offset stays at 0.
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=16,
block_ids=([0],),
block_hashes=[b"a0"],
can_save=True,
)
)
store.batch_is_exist.assert_not_called()
assert thread._saved_offset.get("req-a", 0) == 0
thread._store_pressure_active = False
thread._skip_store_requests.clear()
# The next batch resumes from offset 0, re-covering the chunk skipped under
# pressure (chunk 0) rather than losing it.
thread.add_stored_request("req-a")
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_saves_only_new_masked_chunks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
coord = SimpleNamespace(
lcm_block_size=16,
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
None,
[True, False],
),
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=16,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_masked = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
)
db_masked.set_kv_caches_base_addr([0x2000])
db_masked.set_block_len([256])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_masked],
)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
masked_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
assert full_hashes == [b"a2".hex(), b"a3".hex()]
assert masked_hashes == [b"a2".hex()]
def test_store_sending_thread_only_skips_on_no_available_handle():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -943,7 +1206,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
db = w.token_dbs[0]
token_len = len(block_hashes) * db.block_size
keys = [
key.to_string() for _, _, key in db.process_tokens(token_len, block_hashes)
PoolKey(db.metadata, block_hash.hex()).to_string()
for _, _, block_hash in db.process_tokens(token_len, block_hashes)
]
assert len(keys) == len(block_hashes)
# PUT side: mirrors KVCacheStoreSendingThread's striding slice.
@@ -1047,8 +1311,8 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
[256] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
full_spec = FullAttentionSpec(
@@ -1113,6 +1377,77 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
full_spec = FullAttentionSpec(
block_size=32, num_kv_heads=8, head_size=64, dtype=None
)
swa_spec = SlidingWindowSpec(
block_size=8,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=8,
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
scheduler_block_size=32,
hash_block_size=8,
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=32,
hash_block_size=8,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([512])
db_swa = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=8,
)
db_swa.set_kv_caches_base_addr([0x2000])
db_swa.set_block_len([128])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_swa],
block_size=32,
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._saved_offset["r0"] = 32
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
swa_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
assert full_hashes == [hs[7].hex()]
assert swa_hashes == [hs[7].hex()]
def test_store_sending_thread_kv_events_use_group_chunk_metadata():
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (