[Mooncake] Only check and store new KV cache range (#46412)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -246,6 +246,20 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
assert masks[1] == [False, False, True, True, False, False, True, True]
|
||||
|
||||
|
||||
def test_store_mask_swa_prefix_stable_as_aligned_length_grows():
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
|
||||
shorter = coord.store_mask(64)[1]
|
||||
longer = coord.store_mask(128)[1]
|
||||
|
||||
assert shorter is not None
|
||||
assert longer is not None
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
|
||||
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
|
||||
@@ -355,6 +369,33 @@ def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
|
||||
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
|
||||
|
||||
|
||||
def test_store_mask_suffix_matches_full_mask_tail():
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
|
||||
full = coord.store_mask(128, num_prompt_tokens=100)
|
||||
suffix = coord.store_mask(128, start_token=64, num_prompt_tokens=100)
|
||||
|
||||
for g_idx, cache_group in enumerate(coord.kv_cache_groups):
|
||||
block_size = cache_group.kv_cache_spec.block_size
|
||||
start_chunk = 64 // block_size
|
||||
end_chunk = 128 // block_size
|
||||
full_mask = full[g_idx]
|
||||
if full_mask is None:
|
||||
assert suffix[g_idx] is None
|
||||
else:
|
||||
assert suffix[g_idx] == full_mask[start_chunk:end_chunk]
|
||||
|
||||
|
||||
def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
|
||||
|
||||
shorter = coord.store_mask(64, num_prompt_tokens=100)[1]
|
||||
longer = coord.store_mask(128, num_prompt_tokens=100)[1]
|
||||
|
||||
assert shorter is not None
|
||||
assert longer is not None
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
# ----- Eagle / MTP interaction with load_mask -----
|
||||
|
||||
|
||||
|
||||
@@ -337,5 +337,5 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
assert out[1][0] == 16 and out[1][1] == 32
|
||||
# Each chunk's hash is its last (4th) fine hash, which already chains the
|
||||
# prior three.
|
||||
assert out[0][2].chunk_hash == fine_hashes[3].hex()
|
||||
assert out[1][2].chunk_hash == fine_hashes[7].hex()
|
||||
assert out[0][2].hex() == fine_hashes[3].hex()
|
||||
assert out[1][2].hex() == fine_hashes[7].hex()
|
||||
|
||||
@@ -36,6 +36,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
|
||||
|
||||
class _RecordingBlockHashes:
|
||||
def __init__(self, values: list[bytes]):
|
||||
self.values = values
|
||||
self.accessed: list[int] = []
|
||||
|
||||
def __len__(self):
|
||||
return len(self.values)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx, slice):
|
||||
return [self[i] for i in range(*idx.indices(len(self)))]
|
||||
if idx < 0:
|
||||
idx += len(self.values)
|
||||
if not 0 <= idx < len(self.values):
|
||||
raise IndexError(idx)
|
||||
self.accessed.append(idx)
|
||||
return self.values[idx]
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
@@ -53,6 +72,8 @@ def _make_store_sending_thread(
|
||||
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
|
||||
token_databases: list[ChunkedTokenDatabase] | None = None,
|
||||
block_size: int = 16,
|
||||
tp_rank: int = 0,
|
||||
put_step: int = 1,
|
||||
replicate_config: object | None = None,
|
||||
) -> mooncake_store_worker.KVCacheStoreSendingThread:
|
||||
if coord is None:
|
||||
@@ -67,8 +88,8 @@ def _make_store_sending_thread(
|
||||
token_databases=token_databases,
|
||||
block_size=block_size,
|
||||
coord=coord,
|
||||
tp_rank=0,
|
||||
put_step=1,
|
||||
tp_rank=tp_rank,
|
||||
put_step=put_step,
|
||||
kv_role="kv_producer",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=replicate_config,
|
||||
@@ -391,6 +412,248 @@ def test_store_sending_thread_records_mooncake_metrics():
|
||||
assert stats.data["save_put"][0]["status"] == "ok"
|
||||
|
||||
|
||||
def test_process_tokens_uses_mask_num_as_start_chunk():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=96,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [11]
|
||||
assert results == [(64, 96, bytes([11]))]
|
||||
|
||||
|
||||
def test_process_tokens_applies_chunk_mask_before_hash_access():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=128,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
chunk_mask=[False, True],
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [15]
|
||||
assert results == [(96, 128, bytes([15]))]
|
||||
|
||||
|
||||
def test_process_tokens_applies_stride_before_hash_access():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=128,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
put_step=2,
|
||||
put_step_rank=1,
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [15]
|
||||
assert results == [(96, 128, bytes([15]))]
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_strides_with_local_phase():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
# Under pressure the request is skipped without persisting anything, so the
|
||||
# saved offset stays at 0.
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=16,
|
||||
block_ids=([0],),
|
||||
block_hashes=[b"a0"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
assert thread._saved_offset.get("req-a", 0) == 0
|
||||
|
||||
thread._store_pressure_active = False
|
||||
thread._skip_store_requests.clear()
|
||||
|
||||
# The next batch resumes from offset 0, re-covering the chunk skipped under
|
||||
# pressure (chunk 0) rather than losing it.
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_masked_chunks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
coord = SimpleNamespace(
|
||||
lcm_block_size=16,
|
||||
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
|
||||
None,
|
||||
[True, False],
|
||||
),
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=16,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_masked = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
)
|
||||
db_masked.set_kv_caches_base_addr([0x2000])
|
||||
db_masked.set_block_len([256])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_masked],
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
|
||||
masked_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
|
||||
|
||||
assert full_hashes == [b"a2".hex(), b"a3".hex()]
|
||||
assert masked_hashes == [b"a2".hex()]
|
||||
|
||||
|
||||
def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -943,7 +1206,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
|
||||
db = w.token_dbs[0]
|
||||
token_len = len(block_hashes) * db.block_size
|
||||
keys = [
|
||||
key.to_string() for _, _, key in db.process_tokens(token_len, block_hashes)
|
||||
PoolKey(db.metadata, block_hash.hex()).to_string()
|
||||
for _, _, block_hash in db.process_tokens(token_len, block_hashes)
|
||||
]
|
||||
assert len(keys) == len(block_hashes)
|
||||
# PUT side: mirrors KVCacheStoreSendingThread's striding slice.
|
||||
@@ -1047,8 +1311,8 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
|
||||
[256] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
@@ -1113,6 +1377,77 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
swa_spec = SlidingWindowSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=8,
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
|
||||
scheduler_block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([512])
|
||||
db_swa = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=8,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_swa.set_kv_caches_base_addr([0x2000])
|
||||
db_swa.set_block_len([128])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=32,
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._saved_offset["r0"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
|
||||
swa_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
|
||||
assert full_hashes == [hs[7].hex()]
|
||||
assert swa_hashes == [hs[7].hex()]
|
||||
|
||||
|
||||
def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
|
||||
Reference in New Issue
Block a user