[KV Connector] Support MooncakeStore with hybrid DCP prefix caching (#53324)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: wzhao18 <weizha@nvidia.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -2334,6 +2334,41 @@ def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity():
|
||||
) == (544, 136)
|
||||
|
||||
|
||||
def test_resolve_dcp_kv_block_size_unwraps_uniform_type_specs():
|
||||
attention = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float16,),
|
||||
)
|
||||
wrapped_attention = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={"attention.0": attention, "attention.1": attention},
|
||||
)
|
||||
wrapped_mamba = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={"mamba.0": mamba, "mamba.1": mamba},
|
||||
)
|
||||
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(attention, 4) == 64
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(wrapped_attention, 4) == 64
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(mamba, 4) == 16
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(wrapped_mamba, 4) == 16
|
||||
|
||||
scaled_attention = kv_cache_utils.resolve_dcp_kv_cache_spec(wrapped_attention, 4)
|
||||
assert scaled_attention.block_size == 64
|
||||
assert isinstance(scaled_attention, UniformTypeKVCacheSpecs)
|
||||
assert all(
|
||||
spec.block_size == 64 for spec in scaled_attention.kv_cache_specs.values()
|
||||
)
|
||||
assert kv_cache_utils.resolve_dcp_kv_cache_spec(wrapped_mamba, 4) is wrapped_mamba
|
||||
|
||||
|
||||
def test_multi_run_layer_compact_strides_place_hoisted_heads():
|
||||
"""A layer-compact run region is its own dense allocation: under LHBNC the head
|
||||
groups sit between the layers and the blocks, so a run's block stride is one head
|
||||
|
||||
@@ -19,7 +19,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeLookupResult,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
TailKeyBoundary,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
@@ -121,10 +123,14 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
connector.register_kv_caches(kv_caches)
|
||||
connector.start_load_kv(MagicMock())
|
||||
connector.wait_for_save()
|
||||
result = connector.get_finished(finished_req_ids)
|
||||
invalid_block_ids = connector.get_block_ids_with_load_errors()
|
||||
|
||||
worker.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker.start_load_kv.assert_called_once_with(metadata)
|
||||
worker.wait_for_save.assert_called_once_with(metadata)
|
||||
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result == ({"req-1"}, {"req-2"})
|
||||
worker.get_block_ids_with_load_errors.assert_called_once_with()
|
||||
@@ -359,7 +365,9 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
|
||||
|
||||
# Blocking lookup (non_block defaults to False) runs on the executor and
|
||||
# returns the resolved hit length.
|
||||
assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5
|
||||
result = client.lookup("req0", num_tokens=128, block_hashes=[])
|
||||
assert result is not None
|
||||
assert result.hit_length == 5
|
||||
|
||||
sent_frames = fake_socket.send_multipart.call_args[0][0]
|
||||
assert sent_frames[0] == protocol.LOOKUP_MSG
|
||||
@@ -394,7 +402,7 @@ def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
|
||||
while time.monotonic() < deadline:
|
||||
result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True)
|
||||
if result is not None:
|
||||
return result
|
||||
return result.hit_length
|
||||
time.sleep(0.005)
|
||||
return None
|
||||
|
||||
@@ -502,11 +510,13 @@ def test_get_num_new_matched_tokens_async_defers_then_reports():
|
||||
|
||||
# Lookup ready with a hit -> report need_to_allocate + async-load flag.
|
||||
hit = 3 * block_size
|
||||
mock_client.lookup.return_value = hit
|
||||
boundary = TailKeyBoundary(group_id=0, num_tokens=4 * block_size)
|
||||
mock_client.lookup.return_value = MooncakeLookupResult(hit, (boundary,))
|
||||
need, load_async = sched.get_num_new_matched_tokens(request, 0)
|
||||
assert need == hit
|
||||
assert load_async == sched.load_async
|
||||
assert sched.load_specs["r1"].kvpool_cached_tokens == hit
|
||||
assert sched.load_specs["r1"].tail_key_boundaries == (boundary,)
|
||||
|
||||
|
||||
def test_protocol_tags_are_distinct_and_non_empty():
|
||||
@@ -519,6 +529,20 @@ def test_protocol_tags_are_distinct_and_non_empty():
|
||||
assert protocol.RESP_OK != protocol.RESP_ERR
|
||||
|
||||
|
||||
def test_lookup_response_round_trip_preserves_tail_keys():
|
||||
result = MooncakeLookupResult(
|
||||
hit_length=20,
|
||||
tail_key_boundaries=(
|
||||
TailKeyBoundary(group_id=0, num_tokens=24),
|
||||
TailKeyBoundary(group_id=1, num_tokens=20),
|
||||
),
|
||||
)
|
||||
|
||||
assert protocol.decode_lookup_response(protocol.encode_lookup_response(result)) == (
|
||||
result
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_reset_connector_cache_invokes_connector_reset():
|
||||
"""Cascade test: Scheduler.reset_prefix_cache(reset_connector=True)
|
||||
cascades into MooncakeStoreConnector.reset_cache without dragging in
|
||||
|
||||
@@ -254,10 +254,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
worker.store = store
|
||||
|
||||
# Both groups stored all 4 blocks -> full hit.
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs).hit_length == 64
|
||||
# Exact-multiple prompt: the full hit is re-derived one block lower,
|
||||
# where both groups' stored blocks still cover the SWA window.
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 48
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs).hit_length == 48
|
||||
|
||||
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
|
||||
swa_keys_outside_window = [
|
||||
@@ -270,12 +270,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
|
||||
# SWA window=32 -> only last 2 blocks must be present in SWA group.
|
||||
# Full has all 4. Coordinator should still return 64.
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs).hit_length == 64
|
||||
# Exact-multiple prompt after eviction: the boundary one block lower
|
||||
# needs SWA block 1, which is gone — no usable stored boundary remains
|
||||
# (the pre-fix arithmetic clamp would have returned 48 and livelocked
|
||||
# on load failure -> recompute -> same lookup).
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 0
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs).hit_length == 0
|
||||
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
MooncakeLookupResult,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
@@ -702,9 +703,9 @@ class _StubLookupClient:
|
||||
num_tokens: int,
|
||||
block_hashes: list[bytes],
|
||||
non_block: bool = False,
|
||||
) -> int:
|
||||
) -> MooncakeLookupResult:
|
||||
self.num_tokens.append(num_tokens)
|
||||
return self._hit_tokens
|
||||
return MooncakeLookupResult(self._hit_tokens)
|
||||
|
||||
|
||||
def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
|
||||
|
||||
@@ -31,8 +31,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
MooncakeLookupResult,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
TailKeyBoundary,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
@@ -150,6 +152,7 @@ def _make_load_req(
|
||||
*,
|
||||
token_len: int,
|
||||
vllm_cached_tokens: int = 0,
|
||||
tail_key_boundaries: tuple[TailKeyBoundary, ...] = (),
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
@@ -161,6 +164,7 @@ def _make_load_req(
|
||||
kvpool_cached_tokens=token_len,
|
||||
can_load=True,
|
||||
token_len=token_len,
|
||||
tail_key_boundaries=tail_key_boundaries,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -305,15 +309,16 @@ def _patch_worker_runtime(
|
||||
local_ip: str = "10.0.0.7",
|
||||
tp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
pcp_size: int = 1,
|
||||
dcp_size: int = 1,
|
||||
) -> None:
|
||||
single_rank_group = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
pcp_group = SimpleNamespace(world_size=pcp_size, rank_in_group=0)
|
||||
# DCP groups are contiguous splits of the TP group (see
|
||||
# parallel_state.py), so dcp_rank == tp_rank % dcp_size.
|
||||
dcp_group = SimpleNamespace(world_size=dcp_size, rank_in_group=tp_rank % dcp_size)
|
||||
monkeypatch.setattr(worker, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
monkeypatch.setattr(worker, "get_tensor_model_parallel_world_size", lambda: tp_size)
|
||||
monkeypatch.setattr(worker, "get_pcp_group", lambda: single_rank_group)
|
||||
monkeypatch.setattr(worker, "get_pcp_group", lambda: pcp_group)
|
||||
monkeypatch.setattr(worker, "get_dcp_group", lambda: dcp_group)
|
||||
monkeypatch.setattr(worker, "get_ip", lambda: local_ip)
|
||||
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
|
||||
@@ -1433,6 +1438,29 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
|
||||
store.batch_get_replica_desc.assert_not_called()
|
||||
|
||||
|
||||
def test_recv_thread_keys_chunk_by_lookup_selected_boundary():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
# 32-token hit over 16-token chunks: chunk 1 would default to the hash at
|
||||
# the 32-token boundary (a1); the lookup matched the 48-token one (a2).
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=32,
|
||||
tail_key_boundaries=(TailKeyBoundary(group_id=0, num_tokens=48),),
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
keys = store.batch_get_into_multi_buffers.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
|
||||
|
||||
def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_STORE_TIER_LOG", "1")
|
||||
caplog_vllm.set_level(logging.INFO, logger=worker.logger.name)
|
||||
@@ -1799,6 +1827,59 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
assert w.store_replicate_config.preferred_segment == "10.0.0.7:50053"
|
||||
|
||||
|
||||
def test_worker_scales_uniform_attention_group_under_dcp(tmp_path, monkeypatch):
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=4, dcp_size=4)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"device_name": "",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
inner_specs = {
|
||||
"layer0": FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None
|
||||
),
|
||||
"layer1": FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=None
|
||||
),
|
||||
}
|
||||
uniform_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs=inner_specs,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(list(inner_specs), uniform_spec)],
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(decode_context_parallel_size=4),
|
||||
kv_cache_config,
|
||||
)
|
||||
|
||||
scaled_spec = w._kv_cache_groups[0].kv_cache_spec
|
||||
assert isinstance(scaled_spec, UniformTypeKVCacheSpecs)
|
||||
assert scaled_spec.block_size == 64
|
||||
assert all(spec.block_size == 64 for spec in scaled_spec.kv_cache_specs.values())
|
||||
assert w.coord.attention_groups[0].spec.block_size == 64
|
||||
assert w.token_dbs[0].block_size == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_size", [1, 4])
|
||||
def test_worker_put_striding_covers_every_rank_get_namespace(
|
||||
tmp_path, monkeypatch, dcp_size
|
||||
@@ -2668,12 +2749,12 @@ def test_lookup_rejects_boundary_missing_one_mamba_shard():
|
||||
# 33 tokens for two 16-token blocks: the hit stops below the request end,
|
||||
# so the full-hit re-derivation stays out of the shard accounting.
|
||||
worker.store.batch_is_exist.side_effect = lambda keys: [1] * len(keys)
|
||||
assert worker.lookup(33, [b"h0", b"h1"]) == 32
|
||||
assert worker.lookup(33, [b"h0", b"h1"]).hit_length == 32
|
||||
|
||||
worker.store.batch_is_exist.side_effect = lambda keys: [
|
||||
0 if "tp_rank:1" in k and "group:1" in k else 1 for k in keys
|
||||
]
|
||||
assert worker.lookup(33, [b"h0", b"h1"]) == 0
|
||||
assert worker.lookup(33, [b"h0", b"h1"]).hit_length == 0
|
||||
|
||||
|
||||
def test_lookup_requires_all_dcp_rank_namespaces():
|
||||
@@ -2684,7 +2765,7 @@ def test_lookup_requires_all_dcp_rank_namespaces():
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 0, 1]
|
||||
|
||||
assert worker.lookup(16, [b"a0"]) == 0
|
||||
assert worker.lookup(16, [b"a0"]).hit_length == 0
|
||||
assert worker.store.batch_is_exist.call_args.args[0] == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:1@pcp0@dcp1@pp_rank:0@group:0@6130",
|
||||
@@ -2696,7 +2777,7 @@ def test_lookup_requires_all_dcp_rank_namespaces():
|
||||
def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 0]
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]).hit_length == 32
|
||||
|
||||
|
||||
def test_lookup_partial_tail_uses_hash_alignment():
|
||||
@@ -2736,7 +2817,7 @@ def test_lookup_partial_tail_uses_hash_alignment():
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 0, 0, 1]
|
||||
|
||||
assert worker.lookup(13, [b"h0", b"h1", b"h2"]) == 12
|
||||
assert worker.lookup(13, [b"h0", b"h1", b"h2"]).hit_length == 12
|
||||
|
||||
|
||||
def test_lookup_full_hit_reuses_existing_boundary():
|
||||
@@ -2744,7 +2825,10 @@ def test_lookup_full_hit_reuses_existing_boundary():
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == 16
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == MooncakeLookupResult(
|
||||
hit_length=16,
|
||||
tail_key_boundaries=(TailKeyBoundary(group_id=0, num_tokens=16),),
|
||||
)
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
@@ -2765,10 +2849,144 @@ def test_lookup_full_hit_with_eagle_pops_once_not_twice():
|
||||
# 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop
|
||||
# gives 48; a spurious re-derivation (anchored at 48) would pop again
|
||||
# and return 32.
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]).hit_length == 48
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_plan_resolves_group_tail_keys_from_existing_hashes():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
]
|
||||
worker.hash_block_size = 4
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
for group_id in range(2)
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
use_eagle=True,
|
||||
)
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
hashes = [BlockHash(f"h{i}".encode()) for i in range(6)]
|
||||
present = {
|
||||
(0, bytes(hashes[3])),
|
||||
(0, bytes(hashes[5])),
|
||||
(1, bytes(hashes[4])),
|
||||
# A later Mamba state also exists, but its exact hit-boundary hash
|
||||
# must win and therefore needs no override.
|
||||
(1, bytes(hashes[5])),
|
||||
}
|
||||
|
||||
def exists(keys):
|
||||
return [
|
||||
int(
|
||||
any(
|
||||
f"@group:{group_id}@{block_hash.hex()}" in key
|
||||
for group_id, block_hash in present
|
||||
)
|
||||
)
|
||||
for key in keys
|
||||
]
|
||||
|
||||
worker.store.batch_is_exist.side_effect = exists
|
||||
|
||||
result = worker.lookup(25, hashes)
|
||||
|
||||
# The eagle drop trims the hit to 20 tokens, but the block that survives
|
||||
# truncation is the one keyed at the 24-token boundary (hashes[5]), which
|
||||
# the load path cannot derive from hit_length.
|
||||
assert result.hit_length == 20
|
||||
assert result.tail_key_boundaries == (
|
||||
TailKeyBoundary(group_id=0, num_tokens=24),
|
||||
TailKeyBoundary(group_id=1, num_tokens=20),
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_plan_recovers_tail_key_after_multi_chunk_convergence():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
]
|
||||
worker.hash_block_size = 4
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
for group_id in range(2)
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
_refresh_group_tp_replication_factors(worker)
|
||||
hashes = [BlockHash(f"h{i}".encode()) for i in range(12)]
|
||||
present = {
|
||||
(0, bytes(hashes[3])),
|
||||
(0, bytes(hashes[7])),
|
||||
(0, bytes(hashes[11])),
|
||||
(1, bytes(hashes[4])),
|
||||
}
|
||||
|
||||
def exists(keys):
|
||||
return [
|
||||
int(
|
||||
any(
|
||||
f"@group:{group_id}@{block_hash.hex()}" in key
|
||||
for group_id, block_hash in present
|
||||
)
|
||||
)
|
||||
for key in keys
|
||||
]
|
||||
|
||||
worker.store.batch_is_exist.side_effect = exists
|
||||
|
||||
result = worker.lookup(49, hashes)
|
||||
|
||||
assert result.hit_length == 20
|
||||
assert result.tail_key_boundaries == (
|
||||
TailKeyBoundary(group_id=0, num_tokens=32),
|
||||
TailKeyBoundary(group_id=1, num_tokens=20),
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
"""The motivating livelock: the producer of a 64-token prompt stored
|
||||
only its SWA tail window (blocks 2-3). The old arithmetic clamp turned
|
||||
@@ -2790,7 +3008,7 @@ def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]).hit_length == 0
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
@@ -2811,7 +3029,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]).hit_length == 64
|
||||
|
||||
|
||||
def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
@@ -2865,7 +3083,7 @@ def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
[f"h{i}".encode() for i in range(12)],
|
||||
)
|
||||
|
||||
assert result == 32
|
||||
assert result.hit_length == 32
|
||||
keys = worker.store.batch_is_exist.call_args.args[0]
|
||||
assert len(keys) == 6
|
||||
swa_keys = [key for key in keys if "@group:1@" in key]
|
||||
@@ -2949,6 +3167,24 @@ def test_consumer_starts_send_thread_only_when_put_is_enabled(save_decode_cache)
|
||||
assert (worker.kv_send_thread is not None) == save_decode_cache
|
||||
|
||||
|
||||
def test_start_load_kv_queues_async_load():
|
||||
w = _make_bare_worker()
|
||||
req = _make_load_req(
|
||||
"load-req",
|
||||
[b"h0", b"h1"],
|
||||
token_len=32,
|
||||
)
|
||||
assert req.load_spec is not None
|
||||
req.load_spec.token_len = 0
|
||||
meta = mooncake_store_worker.MooncakeStoreConnectorMetadata(set(), set())
|
||||
meta.add_request(req)
|
||||
|
||||
w.start_load_kv(meta)
|
||||
|
||||
assert req.load_spec.token_len == 32
|
||||
assert w.recv_request_queue.get_nowait() is req
|
||||
|
||||
|
||||
def test_putting_consumer_queues_decode_save():
|
||||
w = _make_bare_worker(kv_role="kv_consumer", save_decode_cache=True)
|
||||
send_thread = MagicMock()
|
||||
@@ -3385,7 +3621,7 @@ def test_lookup_records_mooncake_metrics():
|
||||
result = worker.lookup(33, [b"a0", b"a1"])
|
||||
stats = worker.get_kv_connector_stats()
|
||||
|
||||
assert result == 32
|
||||
assert result.hit_length == 32
|
||||
assert isinstance(stats, MooncakeStoreConnectorStats)
|
||||
assert len(stats.data["lookup_exists"]) == 1
|
||||
assert stats.data["lookup_exists"][0]["num_keys"] == 2
|
||||
|
||||
@@ -465,6 +465,7 @@ def make_kv_cache_config(
|
||||
mamba_enabled: bool = False,
|
||||
sw_size: int = 128,
|
||||
num_blocks: int = 100,
|
||||
mamba_cache_mode: Literal["all", "align", "none"] = "none",
|
||||
) -> KVCacheConfig:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
@@ -498,6 +499,7 @@ def make_kv_cache_config(
|
||||
block_size=block_size,
|
||||
shapes=((16,), (16,)),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user