[KVConnector][DSV4] HMA support for Mooncake store connector (#42828)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -9,12 +9,17 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
connector,
|
||||
worker,
|
||||
connector as mooncake_store_connector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeStoreConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_vllm_config
|
||||
@@ -27,6 +32,17 @@ def _make_vllm_config():
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config() -> KVCacheConfig:
|
||||
"""Single-group full-attention KVCacheConfig — enough for the connector
|
||||
constructor's validate() pass."""
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[KVCacheTensor(size=8192, shared_by=["layer0"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
)
|
||||
|
||||
|
||||
def _make_block_stored() -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[b"hash"],
|
||||
@@ -41,6 +57,7 @@ def _make_block_stored() -> BlockStored:
|
||||
|
||||
def test_scheduler_role_initializes_store_scheduler_only():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -53,75 +70,19 @@ def test_scheduler_role_initializes_store_scheduler_only():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_scheduler.assert_called_once_with(vllm_config)
|
||||
mock_scheduler.assert_called_once_with(vllm_config, kv_cache_config)
|
||||
mock_worker.assert_not_called()
|
||||
assert conn.connector_scheduler is mock_scheduler.return_value
|
||||
assert conn.connector_worker is None
|
||||
|
||||
|
||||
def test_worker_role_initializes_store_worker_on_rank0():
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreScheduler"
|
||||
) as mock_scheduler,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
|
||||
mock_scheduler.assert_not_called()
|
||||
mock_worker.assert_called_once_with(vllm_config)
|
||||
assert conn.connector_scheduler is None
|
||||
assert conn.connector_worker is mock_worker.return_value
|
||||
|
||||
|
||||
def test_worker_role_initializes_on_nonzero_rank():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.rank = 1
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker,
|
||||
):
|
||||
connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
|
||||
mock_worker.assert_called_once_with(vllm_config)
|
||||
|
||||
|
||||
def test_lookup_rpc_path_uses_data_parallel_index_in_dense_dp():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.data_parallel_rank = 0
|
||||
vllm_config.parallel_config.data_parallel_index = 3
|
||||
|
||||
path = worker.get_zmq_rpc_path_lookup(vllm_config)
|
||||
|
||||
assert path.endswith("_dp_rank3")
|
||||
|
||||
|
||||
def test_lookup_rpc_path_uses_local_rank_when_local_engines_only():
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.parallel_config.data_parallel_index = 7
|
||||
vllm_config.parallel_config.data_parallel_rank_local = 1
|
||||
vllm_config.parallel_config.data_parallel_hybrid_lb = True
|
||||
|
||||
path = worker.get_zmq_rpc_path_lookup(vllm_config)
|
||||
|
||||
assert path.endswith("_dp_rank1")
|
||||
assert connector.connector_scheduler is mock_scheduler.return_value
|
||||
assert connector.connector_worker is None
|
||||
|
||||
|
||||
def test_worker_methods_delegate_to_store_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
kv_caches = {"layer0": MagicMock()}
|
||||
metadata = MooncakeStoreConnectorMetadata(set(), set())
|
||||
finished_req_ids = {"req-1"}
|
||||
@@ -133,22 +94,25 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
worker_inst = mock_worker_cls.return_value
|
||||
worker_inst.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
conn.bind_connector_metadata(metadata)
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
conn.register_kv_caches(kv_caches)
|
||||
result = conn.get_finished(finished_req_ids)
|
||||
connector.register_kv_caches(kv_caches)
|
||||
result = connector.get_finished(finished_req_ids)
|
||||
|
||||
worker_inst.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker_inst.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
worker.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result == ({"req-1"}, {"req-2"})
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -157,14 +121,17 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = []
|
||||
assert conn.get_kv_connector_kv_cache_events() is None
|
||||
assert connector.get_kv_connector_kv_cache_events() is None
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
event = _make_block_stored()
|
||||
|
||||
with (
|
||||
@@ -174,12 +141,14 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = [event]
|
||||
kv_events = conn.get_kv_connector_kv_cache_events()
|
||||
kv_events = connector.get_kv_connector_kv_cache_events()
|
||||
|
||||
assert isinstance(kv_events, connector.MooncakeStoreKVEvents)
|
||||
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
|
||||
assert kv_events.get_number_of_workers() == 1
|
||||
assert kv_events.get_all_events() == [event]
|
||||
|
||||
@@ -187,6 +156,7 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
def test_prefer_cross_layer_blocks_from_config():
|
||||
# Default: disabled
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
@@ -194,8 +164,10 @@ def test_prefer_cross_layer_blocks_from_config():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
assert conn.prefer_cross_layer_blocks is False
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert connector.prefer_cross_layer_blocks is False
|
||||
|
||||
# Enabled via config
|
||||
vllm_config_enabled = create_vllm_config(
|
||||
@@ -210,14 +182,15 @@ def test_prefer_cross_layer_blocks_from_config():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn_enabled = connector.MooncakeStoreConnector(
|
||||
vllm_config_enabled, KVConnectorRole.SCHEDULER
|
||||
connector_enabled = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config_enabled, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert conn_enabled.prefer_cross_layer_blocks is True
|
||||
assert connector_enabled.prefer_cross_layer_blocks is True
|
||||
|
||||
|
||||
def test_register_cross_layers_kv_cache_delegates_to_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
@@ -226,18 +199,21 @@ def test_register_cross_layers_kv_cache_delegates_to_worker():
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.WORKER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
fake_tensor = MagicMock()
|
||||
fake_backend = MagicMock()
|
||||
conn.register_cross_layers_kv_cache(fake_tensor, fake_backend)
|
||||
connector.register_cross_layers_kv_cache(fake_tensor, fake_backend)
|
||||
|
||||
worker_inst = mock_worker_cls.return_value
|
||||
worker_inst.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.register_cross_layers_kv_caches.assert_called_once_with(fake_tensor)
|
||||
|
||||
|
||||
def test_update_connector_output_and_take_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
event = _make_block_stored()
|
||||
|
||||
with (
|
||||
@@ -247,12 +223,14 @@ def test_update_connector_output_and_take_events():
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
conn = connector.MooncakeStoreConnector(vllm_config, KVConnectorRole.SCHEDULER)
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
|
||||
kv_events = connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events.add_events([event])
|
||||
conn.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
|
||||
assert conn._kv_cache_events is kv_events
|
||||
assert list(conn.take_events()) == [event]
|
||||
assert conn._kv_cache_events is None
|
||||
assert connector._kv_cache_events is kv_events
|
||||
assert list(connector.take_events()) == [event]
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from math import lcm
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
def _make_coord(groups, hash_block_size, use_eagle=False):
|
||||
"""Construct a coordinator using the natural LCM of group block sizes as
|
||||
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
|
||||
the test fixtures."""
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in groups]
|
||||
scheduler_block_size = lcm(*block_sizes)
|
||||
return MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
|
||||
|
||||
# ----- ExternalCachedBlockPool -----
|
||||
|
||||
|
||||
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
||||
cmap = ExternalCachedBlockPool()
|
||||
h = BlockHash(b"\xaa" * 4)
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
assert len(res) == 2
|
||||
assert res[0] is not cmap.null_block
|
||||
assert res[1] is not cmap.null_block
|
||||
|
||||
|
||||
def test_external_cached_block_pool_hit_all_groups():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
assert len(res) == 2
|
||||
assert res[0] is not cmap.null_block
|
||||
assert res[1] is not cmap.null_block
|
||||
|
||||
|
||||
def test_external_cached_block_pool_miss_one_group():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h))})
|
||||
assert cmap.get_cached_block(h, [0, 1]) is None
|
||||
|
||||
|
||||
def test_external_cached_block_pool_unknown_hash():
|
||||
h_known = BlockHash(b"\x01" * 4)
|
||||
h_unknown = BlockHash(b"\x02" * 4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
|
||||
assert cmap.get_cached_block(h_unknown, [0]) is None
|
||||
|
||||
|
||||
# ----- Helpers -----
|
||||
|
||||
|
||||
def _full(block_size=16, sliding_window=None):
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def _swa(block_size=16, sliding_window=32):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def _hashes(n: int) -> list[BlockHash]:
|
||||
return [BlockHash(bytes([i + 1]) * 4) for i in range(n)]
|
||||
|
||||
|
||||
# ----- Single-group coordinator -----
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_all_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
assert masks[0] == [True, True, True, True]
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_partial_prefix():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 32
|
||||
assert masks[0] == [True, True]
|
||||
|
||||
|
||||
def test_coordinator_single_full_attention_no_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool(set())
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 0
|
||||
assert masks[0] == []
|
||||
|
||||
|
||||
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
|
||||
"""SWA tautological-pool: hit_length spans full prefix, mask is
|
||||
tail-window only."""
|
||||
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4) # 4 chunks * 16 tokens
|
||||
cmap = ExternalCachedBlockPool()
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
|
||||
assert masks[0][-2:] == [True, True]
|
||||
assert all(not m for m in masks[0][:-2])
|
||||
|
||||
|
||||
# ----- Hybrid coordinator (single-group worker, multi-group coordinator) -----
|
||||
|
||||
|
||||
def test_coordinator_hybrid_full_plus_swa_all_hit():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 64
|
||||
|
||||
|
||||
def test_coordinator_hybrid_hole_in_full_clips_both():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
|
||||
exists |= {(1, bytes(h)) for h in hs}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 16
|
||||
|
||||
|
||||
def test_coordinator_group_block_size_double_hash():
|
||||
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
|
||||
hashes merge before pool lookup."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _full(32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32))
|
||||
exists = {(0, bytes(h)) for h in hs}
|
||||
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
assert hit == 64
|
||||
assert hit % 32 == 0
|
||||
|
||||
|
||||
# ----- store_mask -----
|
||||
|
||||
|
||||
def test_store_mask_full_attention_all_true():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True, True, True, True],)
|
||||
|
||||
|
||||
def test_store_mask_zero_aligned_returns_empty_per_group():
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(0)
|
||||
assert masks == ([], [])
|
||||
|
||||
|
||||
def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
|
||||
aligned=64 the SWA group should mark exactly the blocks ending at 32
|
||||
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
|
||||
participate in any future hit."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
# Full-attn: 2 chunks * 32 tokens.
|
||||
assert masks[0] == [True, True]
|
||||
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
|
||||
assert masks[1] == [False, False, False, True, False, False, False, True]
|
||||
|
||||
|
||||
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
|
||||
boundary should now span two SWA tail blocks."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=16)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks[0] == [True, True]
|
||||
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
|
||||
# Boundary at 64: chunks 6 and 7. Others stay False.
|
||||
assert masks[1] == [False, False, True, True, False, False, True, True]
|
||||
|
||||
|
||||
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
|
||||
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
|
||||
(aligned_len=512). Validates that the tile-once strategy produces the
|
||||
expected per-segment tail-window pattern, repeated."""
|
||||
full_mla = _full(block_size=256)
|
||||
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
|
||||
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
|
||||
swa_4_sw16 = _swa(block_size=4, sliding_window=16)
|
||||
swa_8_sw64 = _swa(block_size=8, sliding_window=64)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full_mla),
|
||||
KVCacheGroupSpec(["L1"], swa_64_sw128),
|
||||
KVCacheGroupSpec(["L2"], swa_64_sw512),
|
||||
KVCacheGroupSpec(["L3"], swa_4_sw16),
|
||||
KVCacheGroupSpec(["L4"], swa_8_sw64),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=4)
|
||||
assert coord.lcm_block_size == 256
|
||||
masks = coord.store_mask(512)
|
||||
|
||||
# Full-MLA: 2 chunks of 256, both stored.
|
||||
assert masks[0] == [True, True]
|
||||
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
|
||||
# Per-segment template = [F,F,T,T]; tiled twice.
|
||||
assert masks[1] == [False, False, True, True] * 2
|
||||
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
|
||||
assert masks[2] == [True] * 8
|
||||
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
|
||||
# Last 4 of each 64-chunk segment True.
|
||||
assert masks[3] == ([False] * 60 + [True] * 4) * 2
|
||||
# SWA(8, sw=64): tail = ceil(63/8) = 8; C = 256/8 = 32.
|
||||
# Last 8 of each 32-chunk segment True.
|
||||
assert masks[4] == ([False] * 24 + [True] * 8) * 2
|
||||
|
||||
|
||||
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
|
||||
"""When every non-full-attn group already aligns to lcm_block_size, the
|
||||
fast path returns all-True without invoking find_longest_cache_hit."""
|
||||
full = _full(block_size=64)
|
||||
swa = _swa(block_size=64, sliding_window=128)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=64)
|
||||
assert coord.lcm_block_size == 64
|
||||
masks = coord.store_mask(256)
|
||||
# Every block in every group is True — no sub-lcm filtering possible.
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
|
||||
|
||||
def test_store_mask_fast_path_single_attention_group():
|
||||
"""Two groups sharing the same SWA spec collapse to one attention group;
|
||||
no lcm filter applies, every chunk is True."""
|
||||
swa = _swa(block_size=16, sliding_window=32)
|
||||
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
assert len(coord.attention_groups) == 1
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
@@ -0,0 +1,342 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
|
||||
(SWA + Full) attention config, using a dict-backed mock store."""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker as mooncake_store_worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
|
||||
KVCacheStoreRecvingThread,
|
||||
KVCacheStoreSendingThread,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
|
||||
class _DictStore:
|
||||
"""In-memory MooncakeDistributedStore stand-in."""
|
||||
|
||||
def __init__(self):
|
||||
self._data: dict[str, bytes] = {}
|
||||
|
||||
def setup(self, *_args, **_kwargs):
|
||||
return 0
|
||||
|
||||
def register_buffer(self, addr, length):
|
||||
return 0
|
||||
|
||||
def batch_is_exist(self, keys):
|
||||
return [1 if k in self._data else 0 for k in keys]
|
||||
|
||||
def batch_put_from_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
|
||||
for k in keys:
|
||||
self._data[k] = b"x"
|
||||
return [0] * len(keys)
|
||||
|
||||
def batch_get_into_multi_buffers(self, keys, addrs, sizes, *_args, **_kwargs):
|
||||
return [0 if k in self._data else -1 for k in keys]
|
||||
|
||||
|
||||
def _minimal_vllm_config(cache_block_size=16):
|
||||
cfg = MagicMock()
|
||||
cfg.cache_config.block_size = cache_block_size
|
||||
cfg.cache_config.num_gpu_blocks = 4
|
||||
cfg.cache_config.hash_block_size = None
|
||||
cfg.cache_config.enable_prefix_caching = True
|
||||
cfg.parallel_config.prefill_context_parallel_size = 1
|
||||
cfg.parallel_config.decode_context_parallel_size = 1
|
||||
cfg.parallel_config.pipeline_parallel_size = 1
|
||||
cfg.parallel_config.world_size = 1
|
||||
cfg.parallel_config.rank = 0
|
||||
cfg.parallel_config.data_parallel_rank_local = 0
|
||||
cfg.parallel_config.data_parallel_size_local = 1
|
||||
cfg.kv_transfer_config.kv_role = "kv_both"
|
||||
cfg.kv_transfer_config.kv_connector_extra_config = {}
|
||||
cfg.kv_events_config = None
|
||||
cfg.model_config.model = "/tmp/m"
|
||||
cfg.model_config.use_mla = False
|
||||
cfg.model_config.get_num_layers.return_value = 2
|
||||
cfg.model_config.get_total_num_kv_heads.return_value = 8
|
||||
cfg.model_config.max_model_len = 4096
|
||||
cfg.scheduler_config.max_num_batched_tokens = 8192
|
||||
# Without this, MagicMock's truthy use_eagle() triggers the coordinator's
|
||||
# "use_eagle && nothing annotated → flag all groups" fallback.
|
||||
cfg.speculative_config = None
|
||||
return cfg
|
||||
|
||||
|
||||
def _build_worker_with_dict_store(vllm_config, kv_cache_config, store):
|
||||
"""Build a MooncakeStoreWorker patching all distributed dependencies."""
|
||||
fake_mooncake_store = types.ModuleType("mooncake.store")
|
||||
fake_mooncake_store.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined]
|
||||
fake_mooncake_store.ReplicateConfig = MagicMock # type: ignore[attr-defined]
|
||||
with (
|
||||
patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}),
|
||||
patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg,
|
||||
):
|
||||
sc = MCfg.load_from_env.return_value
|
||||
sc.metadata_server = ""
|
||||
sc.global_segment_size = 1 << 20
|
||||
sc.local_buffer_size = 1 << 20
|
||||
sc.protocol = "tcp"
|
||||
sc.device_name = ""
|
||||
sc.master_server_address = ""
|
||||
sc.enable_offload = False
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_pcp_group"
|
||||
) as mock_pcp,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_dcp_group"
|
||||
) as mock_dcp,
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
|
||||
".store.worker.get_ip",
|
||||
return_value="127.0.0.1",
|
||||
),
|
||||
):
|
||||
mock_pcp.return_value.world_size = 1
|
||||
mock_dcp.return_value.world_size = 1
|
||||
worker = mooncake_store_worker.MooncakeStoreWorker(
|
||||
vllm_config, kv_cache_config=kv_cache_config
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
"""
|
||||
E2E: build a SWA+Full hybrid worker, save all blocks via the sending
|
||||
thread (synchronously), then verify lookup returns the full hit length.
|
||||
Also verify that evicting SWA's early blocks (outside its window) still
|
||||
allows a full hit because the window covers the tail.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=32,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=8192, shared_by=["L0"]),
|
||||
KVCacheTensor(size=8192, shared_by=["L1"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], swa),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.put_step = 1
|
||||
worker.num_kv_head = 8
|
||||
|
||||
# Register kv_caches using mocked thread classes so register_kv_caches
|
||||
# doesn't try to start real background threads (which set ready_event).
|
||||
kv_caches = {
|
||||
"L0": torch.zeros(2, 4, 8, 8, 64),
|
||||
"L1": torch.zeros(2, 4, 8, 8, 64),
|
||||
}
|
||||
|
||||
def _fake_thread_init(*args, **kwargs):
|
||||
"""Mock thread that sets all threading.Event args so waits don't block."""
|
||||
for v in list(args) + list(kwargs.values()):
|
||||
if isinstance(v, threading.Event):
|
||||
v.set()
|
||||
m = MagicMock()
|
||||
m.start = lambda: None
|
||||
return m
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreSendingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
patch.object(
|
||||
mooncake_store_worker,
|
||||
"KVCacheStoreRecvingThread",
|
||||
side_effect=_fake_thread_init,
|
||||
),
|
||||
):
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
# Now build a real sending thread (no .start()) over the worker's token_dbs
|
||||
# and the dict-backed store, so _handle_request runs synchronously.
|
||||
ready = threading.Event()
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=worker.tp_rank,
|
||||
put_step=worker.put_step,
|
||||
kv_role=worker.kv_role,
|
||||
ready_event=ready,
|
||||
enable_kv_event=False,
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
send_thread.add_stored_request("r0")
|
||||
# Put the request in the queue so task_done() doesn't underflow.
|
||||
send_thread.request_queue.put(save_req)
|
||||
req = send_thread.request_queue.get()
|
||||
send_thread._handle_request(req)
|
||||
|
||||
# Point worker.store at the dict store (the worker constructor captured
|
||||
# the MagicMock; replace with the real dict store for lookup).
|
||||
worker.store = store
|
||||
|
||||
# Both groups stored all 4 blocks -> full hit.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
|
||||
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
|
||||
swa_keys_outside_window = [
|
||||
k
|
||||
for k in list(store._data.keys())
|
||||
if "@group:1" in k and (("@" + hs[0].hex()) in k or ("@" + hs[1].hex()) in k)
|
||||
]
|
||||
for k in swa_keys_outside_window:
|
||||
del store._data[k]
|
||||
|
||||
# SWA window=32 -> only last 2 blocks must be present in SWA group.
|
||||
# Full has all 4. Coordinator should still return 64.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
"""Producer stored every block for both groups; consumer must only fetch
|
||||
SWA blocks within the sliding window, not the head."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=32,
|
||||
)
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], swa),
|
||||
]
|
||||
md0 = KeyMetadata("m", 0, 0, 0, 0, group_id=0)
|
||||
md1 = KeyMetadata("m", 0, 0, 0, 0, group_id=1)
|
||||
db_full = ChunkedTokenDatabase(md0, block_size=16, hash_block_size=16)
|
||||
db_swa = ChunkedTokenDatabase(md1, block_size=16, hash_block_size=16)
|
||||
db_full.set_kv_caches_base_addr([0])
|
||||
db_full.set_block_len([1024])
|
||||
db_swa.set_kv_caches_base_addr([1 << 20])
|
||||
db_swa.set_block_len([1024])
|
||||
|
||||
requested_keys: list[str] = []
|
||||
|
||||
class _CapturingStore:
|
||||
def batch_get_into_multi_buffers(self, keys, addrs, sizes):
|
||||
requested_keys.extend(keys)
|
||||
return [0] * len(keys)
|
||||
|
||||
ready = threading.Event()
|
||||
coord = MooncakeStoreCoordinator(
|
||||
groups, scheduler_block_size=16, hash_block_size=16
|
||||
)
|
||||
recv = KVCacheStoreRecvingThread(
|
||||
store=_CapturingStore(),
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=ready,
|
||||
coord=coord,
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
load_spec=LoadSpec(
|
||||
vllm_cached_tokens=0, kvpool_cached_tokens=64, can_load=True, token_len=64
|
||||
),
|
||||
)
|
||||
recv.request_queue.put(req)
|
||||
recv._handle_request(recv.request_queue.get())
|
||||
|
||||
full_keys = [k for k in requested_keys if "@group:0" in k]
|
||||
swa_keys = [k for k in requested_keys if "@group:1" in k]
|
||||
# Full attention: load every block (4).
|
||||
assert len(full_keys) == 4
|
||||
# SWA: only the 2 tail-window blocks (hashes hs[2], hs[3]).
|
||||
assert len(swa_keys) == 2
|
||||
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
|
||||
assert swa_hashes == {hs[2].hex(), hs[3].hex()}
|
||||
|
||||
|
||||
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
|
||||
must merge every 4 fine hashes into one chunk hash via
|
||||
BlockHashListWithBlockSize."""
|
||||
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
|
||||
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
|
||||
db.set_kv_caches_base_addr([0])
|
||||
db.set_block_len([512])
|
||||
# 8 fine-grained hashes (32 tokens at hash_block_size=4) → 2 group chunks.
|
||||
fine_hashes = [BlockHash(bytes([i + 1]) * 4) for i in range(8)]
|
||||
out = list(db.process_tokens(token_len=32, block_hashes=fine_hashes))
|
||||
assert len(out) == 2
|
||||
assert out[0][0] == 0 and out[0][1] == 16
|
||||
assert out[1][0] == 16 and out[1][1] == 32
|
||||
# Each chunk's hash is the concatenation of 4 fine hashes.
|
||||
expected0 = b"".join(fine_hashes[0:4]).hex()
|
||||
expected1 = b"".join(fine_hashes[4:8]).hex()
|
||||
assert out[0][2].chunk_hash == expected0
|
||||
assert out[1][2].chunk_hash == expected1
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
|
||||
|
||||
def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.original_block_size = 16
|
||||
scheduler._block_size = 16
|
||||
scheduler._discard_partial_chunks = True
|
||||
scheduler.load_specs = {}
|
||||
scheduler._preempted_req_ids = set()
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
return scheduler
|
||||
|
||||
|
||||
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([2],)],
|
||||
num_computed_tokens=[44],
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens=(
|
||||
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _add_unfinished_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
token_ids: list[int],
|
||||
block_hashes: list[bytes],
|
||||
prefill_end_tokens: int,
|
||||
) -> None:
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=token_ids,
|
||||
block_hashes=block_hashes,
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=44,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=32,
|
||||
token_ids=token_ids[:44],
|
||||
prefill_end_tokens=prefill_end_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
|
||||
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
|
||||
# yet, so the tracker's token_len does not advance and num_tokens_to_save
|
||||
# stays below chunk_boundary — the save is naturally skipped.
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(44)),
|
||||
block_hashes=[b"h0", b"h1"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=[101, 102, 103])
|
||||
)
|
||||
|
||||
assert meta.requests == []
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.token_len == 44
|
||||
assert tracker.num_saved_tokens == 32
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
|
||||
|
||||
def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=None)
|
||||
)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 48
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.token_len == 48
|
||||
assert tracker.num_saved_tokens == 48
|
||||
@@ -19,7 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker as mooncake_store_worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
@@ -27,20 +30,37 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
)
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
return mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["layer0"], spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
|
||||
|
||||
def _make_store_sending_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
|
||||
token_databases: list[ChunkedTokenDatabase] | None = None,
|
||||
block_size: int = 16,
|
||||
replicate_config: object | None = None,
|
||||
) -> worker.KVCacheStoreSendingThread:
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
)
|
||||
token_database.set_kv_caches_base_addr([0x1000])
|
||||
token_database.set_block_len([256])
|
||||
thread = worker.KVCacheStoreSendingThread(
|
||||
) -> mooncake_store_worker.KVCacheStoreSendingThread:
|
||||
if coord is None:
|
||||
coord = _default_send_coord()
|
||||
if token_databases is None:
|
||||
db = ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([256])
|
||||
token_databases = [db]
|
||||
thread = mooncake_store_worker.KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_database=token_database,
|
||||
block_size=16,
|
||||
token_databases=token_databases,
|
||||
block_size=block_size,
|
||||
coord=coord,
|
||||
tp_rank=0,
|
||||
put_step=1,
|
||||
kv_role="kv_producer",
|
||||
@@ -55,18 +75,27 @@ def _make_store_recving_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
) -> worker.KVCacheStoreRecvingThread:
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
)
|
||||
token_database.set_kv_caches_base_addr([0x1000])
|
||||
token_database.set_block_len([256])
|
||||
thread = worker.KVCacheStoreRecvingThread(
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["layer0"], spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
|
||||
store=store,
|
||||
token_database=token_database,
|
||||
token_databases=[token_database],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
@@ -83,7 +112,7 @@ def _make_load_req(
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=token_len,
|
||||
block_ids=list(range(len(block_hashes))),
|
||||
block_ids=(list(range(len(block_hashes))),),
|
||||
block_hashes=block_hashes,
|
||||
load_spec=LoadSpec(
|
||||
vllm_cached_tokens=vllm_cached_tokens,
|
||||
@@ -98,7 +127,7 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=32,
|
||||
block_ids=[0, 1],
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=block_hashes,
|
||||
can_save=True,
|
||||
original_block_size=16,
|
||||
@@ -149,10 +178,31 @@ def _make_vllm_config(
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
rank=0,
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
),
|
||||
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
"""Minimal single-group KVCacheConfig for topology tests."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
)
|
||||
|
||||
|
||||
@@ -445,9 +495,9 @@ def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert sizes == [[256], [256], [256]]
|
||||
store.batch_get_replica_desc.assert_not_called()
|
||||
@@ -467,9 +517,9 @@ def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
|
||||
token_len=48,
|
||||
)
|
||||
expected_keys = [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
store.batch_get_replica_desc.return_value = {
|
||||
expected_keys[0]: [_ReplicaDesc("memory")],
|
||||
@@ -521,10 +571,10 @@ def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
|
||||
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
|
||||
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
]
|
||||
|
||||
|
||||
@@ -556,14 +606,14 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
|
||||
first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2]
|
||||
second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
base_addr = thread.token_database.kv_caches_base_addr[0]
|
||||
block_len = thread.token_database.block_len[0]
|
||||
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
|
||||
block_len = thread.token_databases[0].block_len[0]
|
||||
assert first_addrs == [[base_addr], [base_addr + block_len]]
|
||||
assert second_addrs == [[base_addr + 2 * block_len]]
|
||||
expected_size = block_len
|
||||
@@ -610,9 +660,9 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
|
||||
|
||||
@@ -654,7 +704,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert not hasattr(w, "_isolate_offload_resources")
|
||||
assert store.setup.call_args.args == (
|
||||
@@ -690,7 +740,7 @@ def test_requester_worker_init_prefers_local_hostname_override(
|
||||
},
|
||||
),
|
||||
)
|
||||
worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert store.setup.call_args.args[0] == "worker-a:50053"
|
||||
|
||||
@@ -718,7 +768,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
@@ -748,7 +798,8 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
extra_config={
|
||||
"preferred_segment": "10.0.0.7:50053",
|
||||
}
|
||||
)
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
@@ -760,6 +811,158 @@ def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
"""Partial chunks past the last lcm boundary aren't stored — cache hits
|
||||
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
# Default coord: single full-attn block_size=16, lcm=16.
|
||||
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=33,
|
||||
block_ids=([0, 1, 2],),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
original_block_size=16,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
assert len(keys) == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_skips_when_token_len_below_lcm():
|
||||
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
|
||||
so neither the existence check nor the put should be issued."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
store = MagicMock()
|
||||
# lcm=64 via single full-attn block_size=64.
|
||||
spec = FullAttentionSpec(block_size=64, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L"], spec)],
|
||||
scheduler_block_size=64,
|
||||
hash_block_size=64,
|
||||
)
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=64,
|
||||
hash_block_size=64,
|
||||
)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([1024])
|
||||
thread = _make_store_sending_thread(
|
||||
store, coord=coord, token_databases=[db], block_size=64
|
||||
)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=[b"a0", b"a1"],
|
||||
can_save=True,
|
||||
original_block_size=64,
|
||||
)
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["r0"] == 0
|
||||
|
||||
|
||||
def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
"""For SWA groups, only blocks within ``sliding_window`` of an
|
||||
lcm-aligned boundary are stored — a block outside every such window
|
||||
can never participate in any future hit (mirrors recv-side masking).
|
||||
|
||||
Setup: full-attn (block_size=32) + SWA (block_size=8, sliding_window=8)
|
||||
→ lcm=32. With token_len=64 there are two LCM boundaries (32, 64);
|
||||
the SWA group should store exactly one block per boundary: the block
|
||||
ending at 32 and the block ending at 64.
|
||||
"""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
|
||||
[256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
swa_spec = SlidingWindowSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=8,
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
|
||||
scheduler_block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([512])
|
||||
db_swa = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=8,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_swa.set_kv_caches_base_addr([0x2000])
|
||||
db_swa.set_block_len([128])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=32,
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
original_block_size=32,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
full_keys = [k for k in keys if "@group:0" in k]
|
||||
swa_keys = [k for k in keys if "@group:1" in k]
|
||||
# Full-attn: 2 blocks (chunks ending at 32 and 64).
|
||||
assert len(full_keys) == 2
|
||||
# SWA: only the two blocks ending at lcm boundaries (32 and 64), i.e.
|
||||
# blocks covering tokens [24,32) and [56,64) — hashes hs[3] and hs[7].
|
||||
assert len(swa_keys) == 2
|
||||
swa_hashes = {k.rsplit("@", 1)[-1] for k in swa_keys}
|
||||
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
|
||||
|
||||
|
||||
def _auto_set_ready_event(*args, **kwargs):
|
||||
"""Side effect for mocked thread constructors that auto-sets ready_event."""
|
||||
for arg in args:
|
||||
@@ -771,37 +974,108 @@ def _auto_set_ready_event(*args, **kwargs):
|
||||
return MagicMock()
|
||||
|
||||
|
||||
def _register_with_mocked_threads(
|
||||
worker: mooncake_store_worker.MooncakeStoreWorker,
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
) -> None:
|
||||
"""Call register_kv_caches with the I/O transfer threads mocked out."""
|
||||
prefix = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker."
|
||||
with (
|
||||
patch(prefix + "KVCacheStoreSendingThread", side_effect=_auto_set_ready_event),
|
||||
patch(prefix + "KVCacheStoreRecvingThread", side_effect=_auto_set_ready_event),
|
||||
):
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
|
||||
def _make_bare_worker(
|
||||
*,
|
||||
num_gpu_blocks: int = 10,
|
||||
block_size: int = 16,
|
||||
kv_role: str = "kv_both",
|
||||
) -> worker.MooncakeStoreWorker:
|
||||
) -> mooncake_store_worker.MooncakeStoreWorker:
|
||||
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
|
||||
|
||||
Sets only the attributes that register_kv_caches() reads so we can
|
||||
test the stride-based layout detection without a real
|
||||
MooncakeDistributedStore.
|
||||
"""
|
||||
w = object.__new__(worker.MooncakeStoreWorker)
|
||||
w.cache_config = MagicMock()
|
||||
w.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
w.store = MagicMock()
|
||||
w.store.register_buffer.return_value = 0
|
||||
w.use_mla = False
|
||||
w.token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=block_size
|
||||
worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
|
||||
worker.cache_config = MagicMock()
|
||||
worker.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
worker.store = MagicMock()
|
||||
worker.store.register_buffer.return_value = 0
|
||||
worker.use_mla = False
|
||||
worker.kv_role = kv_role
|
||||
worker.block_size = block_size
|
||||
worker.tp_rank = 0
|
||||
worker.put_step = 1
|
||||
worker.enable_kv_events = False
|
||||
worker.kv_send_thread = None
|
||||
worker.kv_recv_thread = None
|
||||
worker.tp_size = 1
|
||||
worker.num_kv_head = 1
|
||||
worker.pp_size = 1
|
||||
# Minimal single-full-attention-group config so the coordinator-based
|
||||
# lookup path works (the connector no longer carries a legacy single-group
|
||||
# path; everything flows through the coordinator).
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
w.kv_role = kv_role
|
||||
w.block_size = block_size
|
||||
w.tp_rank = 0
|
||||
w.put_step = 1
|
||||
w.enable_kv_events = False
|
||||
w.disk_offload_buffer_budget_bytes = None
|
||||
w.kv_send_thread = None
|
||||
w.kv_recv_thread = None
|
||||
w.store_replicate_config = SimpleNamespace()
|
||||
return w
|
||||
|
||||
worker.disk_offload_buffer_budget_bytes = None
|
||||
worker.store_replicate_config = SimpleNamespace()
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
|
||||
worker._kv_cache_groups = [group]
|
||||
worker.pcp_size = 1
|
||||
worker.dcp_size = 1
|
||||
worker.hash_block_size = block_size
|
||||
worker.metadata = KeyMetadata("test-model", 0, 0, 0, 0)
|
||||
# Pre-build a single-group token_dbs so lookup-only tests don't have to
|
||||
# call register_kv_caches.
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 0]
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
|
||||
|
||||
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
|
||||
tail. Coordinator-driven lookup returns full prefix even though the
|
||||
pre-window blocks are absent."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
|
||||
)
|
||||
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=worker.hash_block_size,
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -812,34 +1086,17 @@ def _make_bare_worker(
|
||||
def test_register_kv_caches_blocks_first_single_segment():
|
||||
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
|
||||
num_blocks = 10
|
||||
page_size_elements = 64 # elements per block
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
page_size_elements = 64
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Shape: (num_blocks, page_size_elements) — blocks outermost, no outer_dims
|
||||
tensor = torch.zeros(num_blocks, page_size_elements, dtype=torch.float16)
|
||||
_register_with_mocked_threads(worker, {"layer0": tensor})
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreSendingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreRecvingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w.register_kv_caches({"layer0": tensor})
|
||||
|
||||
assert len(w.kv_caches_base_addr) == 1
|
||||
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
|
||||
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
|
||||
assert len(w.block_len) == 1
|
||||
assert w.block_len[0] == expected_block_len
|
||||
|
||||
w.store.register_buffer.assert_called_once_with(
|
||||
db = worker.token_dbs[0]
|
||||
assert db.kv_caches_base_addr == [tensor.untyped_storage().data_ptr()]
|
||||
assert db.block_len == [tensor.untyped_storage().nbytes() // num_blocks]
|
||||
worker.store.register_buffer.assert_called_once_with(
|
||||
tensor.untyped_storage().data_ptr(),
|
||||
tensor.untyped_storage().nbytes(),
|
||||
)
|
||||
@@ -851,8 +1108,7 @@ def test_register_kv_caches_kv_first_two_segments():
|
||||
block_size_tokens = 16
|
||||
num_kv_heads = 4
|
||||
head_size = 8
|
||||
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Shape: (2, num_blocks, block_size, num_kv_heads, head_size) — K/V outermost
|
||||
tensor = torch.zeros(
|
||||
@@ -863,32 +1119,13 @@ def test_register_kv_caches_kv_first_two_segments():
|
||||
head_size,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
_register_with_mocked_threads(worker, {"layer0": tensor})
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreSendingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.KVCacheStoreRecvingThread",
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w.register_kv_caches({"layer0": tensor})
|
||||
|
||||
# K/V-first: dim 0 has stride > page_size, so 2 segments
|
||||
assert len(w.kv_caches_base_addr) == 2
|
||||
assert len(w.block_len) == 2
|
||||
|
||||
el = tensor.element_size()
|
||||
seg_stride = tensor.stride(0) * el # stride of the K/V dim in bytes
|
||||
db = worker.token_dbs[0]
|
||||
seg_stride = tensor.stride(0) * tensor.element_size()
|
||||
base = tensor.untyped_storage().data_ptr()
|
||||
assert w.kv_caches_base_addr[0] == base
|
||||
assert w.kv_caches_base_addr[1] == base + seg_stride
|
||||
assert w.block_len[0] == seg_stride // num_blocks
|
||||
assert w.block_len[1] == seg_stride // num_blocks
|
||||
assert db.kv_caches_base_addr == [base, base + seg_stride]
|
||||
assert db.block_len == [seg_stride // num_blocks] * 2
|
||||
|
||||
|
||||
def test_register_kv_caches_cross_layer_single_segment():
|
||||
@@ -897,7 +1134,7 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
num_layers = 4
|
||||
per_layer_page_elements = 64 # elements per layer per block
|
||||
|
||||
w = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
|
||||
# Cross-layer blocks-first tensor: all layers packed into a single
|
||||
# contiguous block. Shape (num_blocks, num_layers * per_layer_page)
|
||||
@@ -918,10 +1155,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
),
|
||||
):
|
||||
# Use the cross-layer wrapper key, same as register_cross_layers_kv_caches
|
||||
w.register_kv_caches({"__cross_layer__": tensor})
|
||||
worker.register_kv_caches({"__cross_layer__": tensor})
|
||||
|
||||
assert len(w.kv_caches_base_addr) == 1
|
||||
assert w.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
db = worker.token_dbs[0]
|
||||
assert len(db.kv_caches_base_addr) == 1
|
||||
assert db.kv_caches_base_addr[0] == tensor.untyped_storage().data_ptr()
|
||||
|
||||
expected_block_len = tensor.untyped_storage().nbytes() // num_blocks
|
||||
# block_len should be per_layer_page_size * num_layers
|
||||
@@ -929,11 +1167,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
expected_block_len
|
||||
== num_layers * per_layer_page_elements * tensor.element_size()
|
||||
)
|
||||
assert len(w.block_len) == 1
|
||||
assert w.block_len[0] == expected_block_len
|
||||
assert len(db.block_len) == 1
|
||||
assert db.block_len[0] == expected_block_len
|
||||
|
||||
# Also verify via register_cross_layers_kv_caches wrapper
|
||||
w2 = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
worker2 = _make_bare_worker(num_gpu_blocks=num_blocks)
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
@@ -946,10 +1184,11 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
side_effect=_auto_set_ready_event,
|
||||
),
|
||||
):
|
||||
w2.register_cross_layers_kv_caches(tensor)
|
||||
worker2.register_cross_layers_kv_caches(tensor)
|
||||
|
||||
assert w2.kv_caches_base_addr == w.kv_caches_base_addr
|
||||
assert w2.block_len == w.block_len
|
||||
db2 = worker2.token_dbs[0]
|
||||
assert db2.kv_caches_base_addr == db.kv_caches_base_addr
|
||||
assert db2.block_len == db.block_len
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1070,7 +1309,8 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"})
|
||||
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"}),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
# setup() receives global_segment_size=0 and the configured local buffer.
|
||||
@@ -1114,7 +1354,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
|
||||
# setup() receives global_segment_size=4 GiB (rank contributes a segment).
|
||||
assert store.setup.call_args.args == (
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
@@ -76,7 +77,7 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@property
|
||||
@@ -87,6 +88,36 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
== "true"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> None:
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
|
||||
|
||||
unsupported: list[str] = []
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
|
||||
spec = g.kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
|
||||
# Enforce Mamba align mode
|
||||
if isinstance(spec, MambaSpec) and spec.block_size != cache_block_size:
|
||||
unsupported.append(
|
||||
f"group {g_idx}: MambaSpec with block_size="
|
||||
f"{spec.block_size} != cache_config.block_size="
|
||||
f"{cache_block_size} (mamba_cache_mode != 'align')"
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
|
||||
unsupported.append(
|
||||
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
|
||||
)
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -99,6 +130,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self._kv_cache_events: MooncakeStoreKVEvents | None = None
|
||||
|
||||
@@ -106,9 +140,11 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self.connector_worker: MooncakeStoreWorker | None = None
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = MooncakeStoreScheduler(vllm_config)
|
||||
self.connector_scheduler = MooncakeStoreScheduler(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config)
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
|
||||
|
||||
# ============================================================
|
||||
# Scheduler-side methods
|
||||
@@ -146,6 +182,13 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished_all_groups(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
@@ -184,6 +227,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1):
|
||||
self, kv_cache: torch.Tensor, attn_backend: type
|
||||
):
|
||||
assert self.connector_worker is not None
|
||||
assert (
|
||||
self._kv_cache_config is not None
|
||||
and len(self._kv_cache_config.kv_cache_groups) == 1
|
||||
), "Cross-layer KV cache does not supported with hybrid models"
|
||||
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from typing import cast
|
||||
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashList,
|
||||
BlockHashListWithBlockSize,
|
||||
KVCacheBlock,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
spec_manager_map,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
# Dummy placeholder hash for store_mask's template computation.
|
||||
_DUMMY_BLOCK_HASH = BlockHash(b"\x00" * 32)
|
||||
|
||||
|
||||
class ExternalCachedBlockPool:
|
||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||
|
||||
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
|
||||
# ``exists=None`` is used on the recv side where hit_length is already
|
||||
# determined and we just want each spec's manager to apply its own mask.
|
||||
self._exists = exists
|
||||
self.null_block = KVCacheBlock(block_id=0)
|
||||
# Dummy ID 1 for present block for duck-typing.
|
||||
self._present_block = KVCacheBlock(block_id=1)
|
||||
|
||||
def get_cached_block(
|
||||
self,
|
||||
block_hash: BlockHash,
|
||||
group_ids: list[int],
|
||||
) -> list[KVCacheBlock] | None:
|
||||
# Mirrors BlockPool.get_cached_block: hit only when every group_id
|
||||
# (groups sharing a spec) has the hash cached.
|
||||
if self._exists is None:
|
||||
return [self._present_block] * len(group_ids)
|
||||
h = bytes(block_hash)
|
||||
if all((g, h) in self._exists for g in group_ids):
|
||||
return [self._present_block] * len(group_ids)
|
||||
return None
|
||||
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
use_eagle: bool = False,
|
||||
) -> None:
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0 for g in kv_cache_groups
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
f"hash_block_size ({hash_block_size})"
|
||||
)
|
||||
assert all(
|
||||
scheduler_block_size % g.kv_cache_spec.block_size == 0
|
||||
for g in kv_cache_groups
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
self.use_eagle = use_eagle
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
attention_groups: list[
|
||||
tuple[KVCacheSpec, list[int], type[SingleTypeKVCacheManager]]
|
||||
] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = spec_manager_map[type(spec)]
|
||||
for existing_spec, group_ids, existing_cls in attention_groups:
|
||||
if existing_spec == spec:
|
||||
assert manager_cls is existing_cls
|
||||
group_ids.append(i)
|
||||
break
|
||||
else:
|
||||
attention_groups.append((spec, [i], manager_cls))
|
||||
# Full attention first (matches upstream convergence ordering).
|
||||
self.attention_groups = sorted(
|
||||
attention_groups,
|
||||
key=lambda x: not isinstance(x[0], FullAttentionSpec),
|
||||
)
|
||||
self.eagle_attn_group_indices: set[int] = {
|
||||
i
|
||||
for i, (_, group_ids, _) in enumerate(self.attention_groups)
|
||||
if any(self.kv_cache_groups[gid].is_eagle_group for gid in group_ids)
|
||||
}
|
||||
if self.use_eagle and not self.eagle_attn_group_indices:
|
||||
self.eagle_attn_group_indices = set(range(len(self.attention_groups)))
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
) -> tuple[tuple[list[bool], ...], int]:
|
||||
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
|
||||
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
|
||||
recv-side callers skip False slots."""
|
||||
blocks_per_group, hit_length = self._find_hit_blocks(
|
||||
block_hashes, max_length, cached_block_pool
|
||||
)
|
||||
masks = tuple(
|
||||
[blk is not cached_block_pool.null_block for blk in blocks]
|
||||
for blocks in blocks_per_group
|
||||
)
|
||||
return masks, hit_length
|
||||
|
||||
def load_mask(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
token_len: int,
|
||||
) -> tuple[list[bool], ...]:
|
||||
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
|
||||
spec would populate chunk ``i`` locally at length ``token_len``
|
||||
(e.g. SWA / Mamba tail-only).
|
||||
"""
|
||||
masks, _ = self.find_longest_cache_hit(
|
||||
block_hashes, token_len, ExternalCachedBlockPool()
|
||||
)
|
||||
return masks
|
||||
|
||||
def store_mask(self, aligned_token_len: int) -> tuple[list[bool], ...]:
|
||||
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
|
||||
group ``g`` would be populated by some future cache hit at length
|
||||
``L = N * lcm_block_size <= aligned_token_len``.
|
||||
"""
|
||||
assert aligned_token_len % self.lcm_block_size == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"lcm_block_size ({self.lcm_block_size})"
|
||||
)
|
||||
if aligned_token_len == 0:
|
||||
return tuple([] for _ in self.kv_cache_groups)
|
||||
|
||||
num_chunks_per_group = [
|
||||
aligned_token_len // g.kv_cache_spec.block_size
|
||||
for g in self.kv_cache_groups
|
||||
]
|
||||
|
||||
# Fast path: single group or full attn groups or uniform block_sizes
|
||||
if all(
|
||||
isinstance(spec, FullAttentionSpec)
|
||||
or spec.block_size == self.lcm_block_size
|
||||
for spec, _, _ in self.attention_groups
|
||||
):
|
||||
return tuple([True] * n for n in num_chunks_per_group)
|
||||
|
||||
n_segments = aligned_token_len // self.lcm_block_size
|
||||
dummy_hashes: list[BlockHash] = [_DUMMY_BLOCK_HASH] * (
|
||||
self.lcm_block_size // self.hash_block_size
|
||||
)
|
||||
template_masks, _ = self.find_longest_cache_hit(
|
||||
dummy_hashes,
|
||||
max_length=self.lcm_block_size,
|
||||
cached_block_pool=ExternalCachedBlockPool(),
|
||||
)
|
||||
return tuple(
|
||||
list(template_masks[g]) * n_segments
|
||||
for g in range(len(self.kv_cache_groups))
|
||||
)
|
||||
|
||||
def block_hashes_for_spec(
|
||||
self, block_hashes: list[BlockHash], spec: KVCacheSpec
|
||||
) -> BlockHashList:
|
||||
if spec.block_size == self.hash_block_size:
|
||||
return block_hashes
|
||||
return BlockHashListWithBlockSize(
|
||||
block_hashes, self.hash_block_size, spec.block_size
|
||||
)
|
||||
|
||||
def _find_hit_blocks(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
if len(self.attention_groups) == 1:
|
||||
spec, group_ids, manager_cls = self.attention_groups[0]
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes,
|
||||
max_length=max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=(0 in self.eagle_attn_group_indices),
|
||||
alignment_tokens=spec.block_size,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
||||
blocks_by_group[gid] = blks
|
||||
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
|
||||
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
hit_length = max_length
|
||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||
|
||||
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
||||
self.attention_groups[0][0], FullAttentionSpec
|
||||
)
|
||||
eagle_verified: set[int] = set()
|
||||
|
||||
while True:
|
||||
curr_hit_length = hit_length
|
||||
|
||||
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
|
||||
cached = hit_blocks_by_group[group_ids[0]]
|
||||
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
||||
curr_hit_length = (
|
||||
curr_hit_length // spec.block_size * spec.block_size
|
||||
)
|
||||
continue
|
||||
|
||||
use_eagle = (
|
||||
idx in self.eagle_attn_group_indices and idx not in eagle_verified
|
||||
)
|
||||
_max_length = curr_hit_length
|
||||
if use_eagle:
|
||||
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes,
|
||||
max_length=_max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
)
|
||||
_new_hit_length = len(hit_blocks[0]) * spec.block_size
|
||||
if use_eagle:
|
||||
eagle_verified.add(idx)
|
||||
elif _new_hit_length < curr_hit_length:
|
||||
eagle_verified.clear()
|
||||
curr_hit_length = _new_hit_length
|
||||
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
||||
hit_blocks_by_group[gid] = blocks
|
||||
|
||||
if curr_hit_length >= hit_length:
|
||||
break
|
||||
hit_length = curr_hit_length
|
||||
if is_simple_hybrid:
|
||||
break
|
||||
|
||||
# Truncate full-attention hit_blocks to final converged length;
|
||||
# other specs already trim themselves inside their hit logic.
|
||||
spec0, group_ids0, _ = self.attention_groups[0]
|
||||
if isinstance(spec0, FullAttentionSpec):
|
||||
num_blocks = hit_length // spec0.block_size
|
||||
for gid in group_ids0:
|
||||
full_blks = hit_blocks_by_group[gid]
|
||||
assert full_blks is not None
|
||||
del full_blks[num_blocks:]
|
||||
|
||||
return (
|
||||
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
||||
hit_length,
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
@@ -15,7 +15,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -29,6 +32,7 @@ class KeyMetadata:
|
||||
pcp_rank: int
|
||||
dcp_rank: int
|
||||
pp_rank: int
|
||||
group_id: int = 0
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
@@ -46,6 +50,7 @@ class PoolKey:
|
||||
self.key_metadata.pcp_rank,
|
||||
self.key_metadata.dcp_rank,
|
||||
self.key_metadata.pp_rank,
|
||||
self.key_metadata.group_id,
|
||||
self.chunk_hash,
|
||||
)
|
||||
)
|
||||
@@ -57,6 +62,7 @@ class PoolKey:
|
||||
f"@pcp{self.key_metadata.pcp_rank}"
|
||||
f"@dcp{self.key_metadata.dcp_rank}"
|
||||
f"@pp_rank:{self.key_metadata.pp_rank}"
|
||||
f"@group:{self.key_metadata.group_id}"
|
||||
f"@{self.chunk_hash}"
|
||||
)
|
||||
|
||||
@@ -64,9 +70,20 @@ class PoolKey:
|
||||
class ChunkedTokenDatabase:
|
||||
"""Maps token positions to store keys and GPU memory addresses."""
|
||||
|
||||
def __init__(self, metadata: KeyMetadata, block_size: int):
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
):
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
|
||||
@@ -77,9 +94,6 @@ class ChunkedTokenDatabase:
|
||||
self.kv_caches_base_addr = kv_caches_base_addr
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
for length in block_len:
|
||||
if length % self.block_size != 0:
|
||||
raise ValueError(f"block_len {length} % {self.block_size} != 0")
|
||||
self.block_len = block_len
|
||||
|
||||
def prepare_value(
|
||||
@@ -96,7 +110,8 @@ class ChunkedTokenDatabase:
|
||||
length = len(self.block_len)
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * self.block_len[index % length]
|
||||
size = self.block_len[index % length] // self.block_size * (end - start)
|
||||
assert (end - start) % self.block_size == 0
|
||||
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
return addr_list, size_list, block_id
|
||||
@@ -104,38 +119,34 @@ class ChunkedTokenDatabase:
|
||||
def process_tokens(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash] | list[str],
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
) -> Iterable[tuple[int, int, PoolKey]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens.
|
||||
block_hashes: Block hashes for each block.
|
||||
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
|
||||
When ``block_size > hash_block_size`` consecutive hashes are merged
|
||||
up to the group's ``block_size`` via ``BlockHashListWithBlockSize``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
"""
|
||||
if not block_hashes:
|
||||
return
|
||||
if not isinstance(block_hashes[0], str):
|
||||
block_hashes = [
|
||||
h.hex() # type: ignore[union-attr]
|
||||
for h in block_hashes
|
||||
]
|
||||
for chunk_id, hash_val in enumerate(block_hashes):
|
||||
if self.block_size == self.hash_block_size:
|
||||
chunk_hashes: Iterable[BlockHash] = block_hashes
|
||||
else:
|
||||
chunk_hashes = BlockHashListWithBlockSize(
|
||||
block_hashes, self.hash_block_size, self.block_size
|
||||
)
|
||||
for chunk_id, h in enumerate(chunk_hashes):
|
||||
start_idx = chunk_id * self.block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
if start_idx < mask_num:
|
||||
continue
|
||||
else:
|
||||
yield (
|
||||
start_idx,
|
||||
end_idx,
|
||||
self._make_key_by_hash(
|
||||
hash_val # type: ignore[arg-type]
|
||||
),
|
||||
)
|
||||
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -154,7 +165,7 @@ class RequestTracker:
|
||||
|
||||
req_id: str
|
||||
token_len: int
|
||||
allocated_block_ids: list[int]
|
||||
allocated_block_ids: tuple[list[int], ...]
|
||||
num_saved_tokens: int = 0
|
||||
token_ids: list[int] | None = None
|
||||
# Snapshot of the prefill range length at tracker creation time.
|
||||
@@ -166,15 +177,18 @@ class RequestTracker:
|
||||
self,
|
||||
new_block_ids: tuple[list[int], ...] | list[int],
|
||||
) -> None:
|
||||
if len(new_block_ids) == 0:
|
||||
new_block_ids = []
|
||||
elif isinstance(new_block_ids, tuple):
|
||||
new_block_ids = new_block_ids[0]
|
||||
elif isinstance(new_block_ids, list):
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Unsupported new_block_ids type {type(new_block_ids)}")
|
||||
self.allocated_block_ids.extend(new_block_ids)
|
||||
# Backward-compat: accept a single list (broadcast to single group).
|
||||
if isinstance(new_block_ids, list):
|
||||
new_block_ids = (new_block_ids,)
|
||||
if len(new_block_ids) != len(self.allocated_block_ids):
|
||||
raise ValueError(
|
||||
f"Group count mismatch: tracker has "
|
||||
f"{len(self.allocated_block_ids)} groups, update has "
|
||||
f"{len(new_block_ids)}"
|
||||
)
|
||||
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
|
||||
if new:
|
||||
existing.extend(new)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -183,7 +197,7 @@ class ReqMeta:
|
||||
|
||||
req_id: str
|
||||
token_len_chunk: int
|
||||
block_ids: list[int]
|
||||
block_ids: tuple[list[int], ...]
|
||||
block_hashes: list[BlockHash]
|
||||
|
||||
can_save: bool | None = None
|
||||
|
||||
@@ -22,7 +22,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -45,7 +47,11 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
|
||||
class MooncakeStoreScheduler:
|
||||
"""Scheduler-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
@@ -56,11 +62,11 @@ class MooncakeStoreScheduler:
|
||||
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.original_block_size = vllm_config.cache_config.block_size
|
||||
self._block_size = vllm_config.cache_config.block_size
|
||||
if self.pcp_size > 1:
|
||||
self._block_size *= self.pcp_size
|
||||
if self.dcp_size > 1:
|
||||
self._block_size *= self.dcp_size
|
||||
# LCM for multi-group HMA; bs * pcp * dcp for single-group. Matches
|
||||
# the engine's own scheduler block size by construction.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
self._discard_partial_chunks = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
@@ -72,7 +78,7 @@ class MooncakeStoreScheduler:
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._preempted_req_ids: set[str] = set() # preempted requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, list[int]]] = {}
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
@@ -126,9 +132,9 @@ class MooncakeStoreScheduler:
|
||||
num_external_tokens: int,
|
||||
):
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: list[int] = []
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids()[0]
|
||||
local_block_ids = blocks.get_block_ids()
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
@@ -189,11 +195,12 @@ class MooncakeStoreScheduler:
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
if not isinstance(request.block_ids[0], list):
|
||||
unfolded_block_ids = request.block_ids.copy()
|
||||
if isinstance(request.block_ids, tuple):
|
||||
# Multi-group: preserve per-group structure.
|
||||
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
|
||||
else:
|
||||
# TODO: support HMA
|
||||
unfolded_block_ids = request.block_ids[0].copy()
|
||||
# Single-group legacy: list[int] -> 1-tuple.
|
||||
unfolded_block_ids = (request.block_ids.copy(),)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
@@ -237,9 +244,9 @@ class MooncakeStoreScheduler:
|
||||
if req_id in self._preempted_req_ids:
|
||||
# Resumed after preemption
|
||||
if isinstance(new_block_ids, tuple):
|
||||
block_ids_list = new_block_ids[0].copy()
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
block_ids_list = new_block_ids.copy()
|
||||
new_block_ids = (new_block_ids.copy(),)
|
||||
self._preempted_req_ids.discard(req_id)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
@@ -254,7 +261,7 @@ class MooncakeStoreScheduler:
|
||||
request_tracker = RequestTracker(
|
||||
req_id=req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=block_ids_list,
|
||||
allocated_block_ids=new_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
@@ -361,7 +368,7 @@ class MooncakeStoreScheduler:
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Determine whether to delay freeing blocks for async save."""
|
||||
if self.kv_role == "kv_consumer":
|
||||
@@ -370,11 +377,12 @@ class MooncakeStoreScheduler:
|
||||
assert tracker is not None
|
||||
if tracker.num_saved_tokens <= 0:
|
||||
return False, None
|
||||
delay_free_blocks = len(block_ids) > 0
|
||||
total_blocks = sum(len(g) for g in block_ids)
|
||||
delay_free_blocks = total_blocks > 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"Delaying free of %d blocks for request %s",
|
||||
len(block_ids),
|
||||
total_blocks,
|
||||
request.request_id,
|
||||
)
|
||||
return delay_free_blocks, None
|
||||
|
||||
@@ -10,6 +10,7 @@ Includes the store worker, transfer threads, lookup server,
|
||||
and MooncakeDistributedStore integration.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
@@ -36,15 +37,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
get_mooncake_dp_engine_index,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -321,7 +332,7 @@ class KVTransferThread(threading.Thread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
@@ -332,7 +343,7 @@ class KVTransferThread(threading.Thread):
|
||||
self.ready_event = ready_event
|
||||
self.block_size = block_size
|
||||
self.tp_rank = tp_rank
|
||||
self.token_database = token_database
|
||||
self.token_databases = token_databases
|
||||
self.done_task_lock = threading.Lock()
|
||||
self.request_queue: queue.Queue[Any] = queue.Queue()
|
||||
self.finished_requests: set[str] = set()
|
||||
@@ -385,7 +396,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
coord: MooncakeStoreCoordinator,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
put_step: int,
|
||||
@@ -396,13 +408,14 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
token_database,
|
||||
token_databases,
|
||||
block_size,
|
||||
tp_rank,
|
||||
ready_event,
|
||||
name="KVCacheStoreSendingThread",
|
||||
)
|
||||
self.put_step = put_step
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.stored_requests: defaultdict[str, int] = defaultdict(int)
|
||||
self.enable_kv_event = enable_kv_event
|
||||
@@ -449,14 +462,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
return True
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
token_len = req_meta.token_len_chunk
|
||||
block_ids = req_meta.block_ids
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if req_id not in self.stored_requests:
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
if token_len == 0:
|
||||
self.dec_stored_request(req_id)
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk offloading "
|
||||
@@ -467,23 +486,34 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
|
||||
starts = []
|
||||
ends = []
|
||||
keys = []
|
||||
# Within each lcm region only per-spec relevant chunks are loaded
|
||||
# (e.g., SWA or linear attn), so mask out irrelevant chunks
|
||||
store_masks = self.coord.store_mask(token_len)
|
||||
starts: list[int] = []
|
||||
ends: list[int] = []
|
||||
keys: list[str] = []
|
||||
block_hashes: list[BlockHash] = []
|
||||
for index, (start, end, key) in enumerate(
|
||||
self.token_database.process_tokens(token_len, req_meta.block_hashes)
|
||||
):
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(req_meta.block_hashes[index])
|
||||
group_indices: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = store_masks[g_idx]
|
||||
for chunk_idx, (start, end, key) in enumerate(
|
||||
db.process_tokens(token_len, req_meta.block_hashes)
|
||||
):
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(req_meta.block_hashes[chunk_idx])
|
||||
group_indices.append(g_idx)
|
||||
|
||||
# Apply put_step striding for TP
|
||||
starts = starts[self.tp_rank % self.put_step :: self.put_step]
|
||||
ends = ends[self.tp_rank % self.put_step :: self.put_step]
|
||||
keys = keys[self.tp_rank % self.put_step :: self.put_step]
|
||||
block_hashes = block_hashes[self.tp_rank % self.put_step :: self.put_step]
|
||||
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
|
||||
starts = starts[sl]
|
||||
ends = ends[sl]
|
||||
keys = keys[sl]
|
||||
block_hashes = block_hashes[sl]
|
||||
group_indices = group_indices[sl]
|
||||
|
||||
if not keys:
|
||||
self.dec_stored_request(req_id)
|
||||
@@ -501,38 +531,37 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
block_hashes = [block_hashes[i] for i in missing_indices]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
|
||||
logger.debug(
|
||||
"Storing KV cache for %d out of %d blocks "
|
||||
"(missing_count=%d) for request %s",
|
||||
"Storing KV cache for %d blocks (groups=%s) for request %s",
|
||||
len(keys),
|
||||
token_len // self.block_size,
|
||||
len(missing_indices),
|
||||
set(group_indices),
|
||||
req_id,
|
||||
)
|
||||
|
||||
addrs = []
|
||||
sizes = []
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
prev_key = None
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
|
||||
|
||||
for index, start in enumerate(starts):
|
||||
addr, size, _ = self.token_database.prepare_value(
|
||||
start, ends[index], block_ids
|
||||
)
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
):
|
||||
db = self.token_databases[g_idx]
|
||||
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
|
||||
addrs.append(addr)
|
||||
sizes.append(size)
|
||||
|
||||
if self.enable_kv_event:
|
||||
token_ids = (
|
||||
req_meta.token_ids[start : ends[index]]
|
||||
if req_meta.token_ids is not None
|
||||
else None
|
||||
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None
|
||||
)
|
||||
stored_event = BlockStored(
|
||||
block_hashes=[new_block_hashes[index]],
|
||||
parent_block_hash=prev_key,
|
||||
block_hashes=[new_block_hashes[idx]],
|
||||
parent_block_hash=prev_key_per_group.get(g_idx),
|
||||
token_ids=token_ids,
|
||||
block_size=req_meta.original_block_size,
|
||||
lora_id=None,
|
||||
@@ -540,7 +569,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
lora_name=None,
|
||||
)
|
||||
stored_events.append(stored_event)
|
||||
prev_key = new_block_hashes[index]
|
||||
prev_key_per_group[g_idx] = new_block_hashes[idx]
|
||||
|
||||
if current_event is not None:
|
||||
current_event.synchronize()
|
||||
@@ -600,7 +629,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
token_database: ChunkedTokenDatabase,
|
||||
coord: MooncakeStoreCoordinator,
|
||||
token_databases: list[ChunkedTokenDatabase],
|
||||
block_size: int,
|
||||
tp_rank: int,
|
||||
ready_event: threading.Event,
|
||||
@@ -608,7 +638,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
token_database,
|
||||
token_databases,
|
||||
block_size,
|
||||
tp_rank,
|
||||
ready_event,
|
||||
@@ -622,6 +652,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
disk_offload_buffer_budget_bytes
|
||||
)
|
||||
)
|
||||
self.coord = coord
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
|
||||
@@ -632,32 +663,31 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
* self.block_size
|
||||
)
|
||||
|
||||
addr_list = []
|
||||
size_list = []
|
||||
key_list = []
|
||||
for start, end, key in self.token_database.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
addr, size, _ = self.token_database.prepare_value(
|
||||
start, end, req_meta.block_ids
|
||||
)
|
||||
key_list.append(key.to_string())
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
# Skip chunks the consumer's per-group spec wouldn't populate
|
||||
# locally (e.g. SWA pre-window) even if the producer stored them.
|
||||
load_mask_per_group = self.coord.load_mask(req_meta.block_hashes, token_len)
|
||||
|
||||
addr_list: list[list[int]] = []
|
||||
size_list: list[list[int]] = []
|
||||
key_list: list[str] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
for start, end, key in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
chunk_idx = start // db.block_size
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
|
||||
key_list.append(key.to_string())
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
|
||||
# Rotate lists by tp_rank for load balancing
|
||||
key_list_c = (
|
||||
key_list[self.tp_rank % len(key_list) :]
|
||||
+ key_list[: self.tp_rank % len(key_list)]
|
||||
)
|
||||
addr_list_c = (
|
||||
addr_list[self.tp_rank % len(addr_list) :]
|
||||
+ addr_list[: self.tp_rank % len(addr_list)]
|
||||
)
|
||||
size_list_c = (
|
||||
size_list[self.tp_rank % len(size_list) :]
|
||||
+ size_list[: self.tp_rank % len(size_list)]
|
||||
)
|
||||
rotation = self.tp_rank % len(key_list)
|
||||
key_list_c = key_list[rotation:] + key_list[:rotation]
|
||||
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
|
||||
size_list_c = size_list[rotation:] + size_list[:rotation]
|
||||
|
||||
load_batches = [(key_list_c, addr_list_c, size_list_c)]
|
||||
if self.usable_disk_offload_buffer_budget_bytes is not None:
|
||||
@@ -737,7 +767,11 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
class MooncakeStoreWorker:
|
||||
"""Worker-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
try:
|
||||
from mooncake.store import ( # type: ignore
|
||||
MooncakeDistributedStore,
|
||||
@@ -771,11 +805,9 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.original_block_size = self.cache_config.block_size
|
||||
self.block_size = self.cache_config.block_size
|
||||
if self.pcp_size > 1:
|
||||
self.block_size *= self.pcp_size
|
||||
if self.dcp_size > 1:
|
||||
self.block_size *= self.dcp_size
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.num_layers = model_config.get_num_layers(parallel_config)
|
||||
|
||||
self.use_mla = False
|
||||
@@ -806,8 +838,6 @@ class MooncakeStoreWorker:
|
||||
pp_rank=self.pp_rank,
|
||||
)
|
||||
|
||||
self.token_database = ChunkedTokenDatabase(self.metadata, self.block_size)
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_env()
|
||||
extra_config = (
|
||||
@@ -892,6 +922,45 @@ class MooncakeStoreWorker:
|
||||
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
|
||||
self.finished_store_req: set[str] = set()
|
||||
|
||||
self._kv_cache_config = kv_cache_config
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
# self.block_size (= scheduler_block_size) so the coordinator's
|
||||
# ``block_size % hash_block_size == 0`` invariant holds.
|
||||
groups = list(kv_cache_config.kv_cache_groups)
|
||||
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
|
||||
g = groups[0]
|
||||
groups = [
|
||||
dataclasses.replace(
|
||||
g,
|
||||
kv_cache_spec=dataclasses.replace(
|
||||
g.kv_cache_spec, block_size=self.block_size
|
||||
),
|
||||
)
|
||||
]
|
||||
self._kv_cache_groups: list[KVCacheGroupSpec] = groups
|
||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||
use_eagle = bool(
|
||||
spec_cfg.use_eagle()
|
||||
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
|
||||
else False
|
||||
)
|
||||
self.coord = MooncakeStoreCoordinator(
|
||||
self._kv_cache_groups,
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
# One ChunkedTokenDatabase per group; addresses populated in
|
||||
# register_kv_caches once the kv-cache layout is known.
|
||||
self.token_dbs: list[ChunkedTokenDatabase] = [
|
||||
ChunkedTokenDatabase(
|
||||
dataclasses.replace(self.metadata, group_id=g_idx),
|
||||
g.kv_cache_spec.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
)
|
||||
for g_idx, g in enumerate(self._kv_cache_groups)
|
||||
]
|
||||
|
||||
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Register a cross-layers KV cache tensor.
|
||||
|
||||
@@ -901,93 +970,86 @@ class MooncakeStoreWorker:
|
||||
"""
|
||||
self.register_kv_caches({"__cross_layer__": kv_cache})
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
|
||||
) -> None:
|
||||
"""Register KV cache tensors and start transfer threads."""
|
||||
# TODO(yifan): we haven't supported HMA yet.
|
||||
first_kv_cache = next(iter(kv_caches.values()))
|
||||
if not kv_caches:
|
||||
logger.warning("No KV caches to offload.")
|
||||
return
|
||||
|
||||
# Resolve each entry to a representative tensor for storage
|
||||
# deduplication. For attention layers the value is already a tensor;
|
||||
# for Mamba layers it is a list of tensors that all share the same
|
||||
# underlying raw storage, so we take the first one.
|
||||
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
|
||||
assert isinstance(v, torch.Tensor | list)
|
||||
return v if isinstance(v, torch.Tensor) else v[0]
|
||||
|
||||
# num_blocks from cache_config is authoritative (set after
|
||||
# profiling, before KV cache allocation).
|
||||
assert self.cache_config.num_gpu_blocks is not None
|
||||
self.num_blocks = self.cache_config.num_gpu_blocks
|
||||
|
||||
# Detect the KV cache memory layout using the stride-based
|
||||
# approach from simple_kv_offload/worker.py.
|
||||
#
|
||||
# The physical layout varies across attention backends:
|
||||
# FlashAttn/ROCm : (2, num_blocks, ...) → K/V outermost
|
||||
# FlashInfer/MLA : (num_blocks, ...) → blocks outermost
|
||||
#
|
||||
# We derive page_size_bytes = storage.nbytes() // num_blocks,
|
||||
# then classify dims: any dim whose byte-stride exceeds
|
||||
# page_size_bytes must be an outer segment dim (e.g. the K/V
|
||||
# dim of size 2). For those backends we register each segment
|
||||
# (K, V) as a separate base-address so that the per-block
|
||||
# offset arithmetic in prepare_value() stays correct.
|
||||
storage = first_kv_cache.untyped_storage()
|
||||
el = first_kv_cache.element_size()
|
||||
page_size_bytes = storage.nbytes() // self.num_blocks
|
||||
outer_dims = [
|
||||
d
|
||||
for d in range(first_kv_cache.ndim)
|
||||
if first_kv_cache.stride(d) * el > page_size_bytes
|
||||
]
|
||||
|
||||
# Register buffers with the store (deduplicate shared storages)
|
||||
# and record per-segment base addresses for every layer.
|
||||
seen_ptrs: set[int] = set()
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
|
||||
for cache in kv_caches.values():
|
||||
for value in kv_caches.values():
|
||||
cache = _repr_tensor(value)
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
if base_addr in seen_ptrs:
|
||||
continue
|
||||
seen_ptrs.add(base_addr)
|
||||
region_len = cache_storage.nbytes()
|
||||
|
||||
if base_addr not in seen_ptrs:
|
||||
seen_ptrs.add(base_addr)
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
|
||||
# Detect layout via stride: a dim whose byte-stride exceeds
|
||||
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
|
||||
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
|
||||
# outermost layout has no such dim and yields a single segment.
|
||||
el = cache.element_size()
|
||||
page_size_bytes = region_len // self.num_blocks
|
||||
outer_dims = [
|
||||
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
|
||||
]
|
||||
if not outer_dims:
|
||||
# Blocks-first layout (FlashInfer / MLA): one segment.
|
||||
self.kv_caches_base_addr.append(base_addr)
|
||||
self.block_len.append(page_size_bytes)
|
||||
addrs.append(base_addr)
|
||||
block_lens.append(page_size_bytes)
|
||||
else:
|
||||
# K/V-first layout (FlashAttn / ROCm): split segments.
|
||||
seg_stride = cache.stride(outer_dims[0]) * el
|
||||
for idx in range(cache.shape[outer_dims[0]]):
|
||||
self.kv_caches_base_addr.append(base_addr + idx * seg_stride)
|
||||
self.block_len.append(seg_stride // self.num_blocks)
|
||||
addrs.append(base_addr + idx * seg_stride)
|
||||
block_lens.append(seg_stride // self.num_blocks)
|
||||
|
||||
logger.info(
|
||||
"Registering KV_Caches. use_mla: %s, shape %s, "
|
||||
"num_blocks: %d, block_len: %s, "
|
||||
"per_key_bytes: %d, "
|
||||
"num_segments: %d",
|
||||
self.use_mla,
|
||||
first_kv_cache.shape,
|
||||
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
|
||||
len(self.token_dbs),
|
||||
len(addrs),
|
||||
self.num_blocks,
|
||||
list(set(self.block_len)),
|
||||
sum(self.block_len),
|
||||
len(self.kv_caches_base_addr),
|
||||
)
|
||||
|
||||
self.token_database.set_kv_caches_base_addr(self.kv_caches_base_addr)
|
||||
self.token_database.set_block_len(self.block_len)
|
||||
for db in self.token_dbs:
|
||||
db.set_kv_caches_base_addr(addrs)
|
||||
db.set_block_len(block_lens)
|
||||
|
||||
# Start transfer threads
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
ready_event_sending = threading.Event()
|
||||
self.kv_send_thread = KVCacheStoreSendingThread(
|
||||
self.store,
|
||||
self.token_database,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
self.put_step,
|
||||
@@ -1001,7 +1063,8 @@ class MooncakeStoreWorker:
|
||||
ready_event_recving = threading.Event()
|
||||
self.kv_recv_thread = KVCacheStoreRecvingThread(
|
||||
self.store,
|
||||
self.token_database,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
ready_event_recving,
|
||||
@@ -1122,62 +1185,55 @@ class MooncakeStoreWorker:
|
||||
|
||||
return finished_sending
|
||||
|
||||
def lookup(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
) -> int:
|
||||
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Checks across all TP ranks and PP ranks.
|
||||
"""
|
||||
end = 0
|
||||
keys: list[str] = []
|
||||
if not block_hashes or token_len <= 0:
|
||||
return 0
|
||||
|
||||
# Build per-(group, hash) candidate keys expanded across TP/PP.
|
||||
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
|
||||
candidate_keys: list[str] = []
|
||||
candidate_meta: list[tuple[int, bytes]] = []
|
||||
tp_count = min(self.tp_size, self.num_kv_head)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
spec_block_size = db.block_size
|
||||
group_hashes = self.coord.block_hashes_for_spec(
|
||||
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
|
||||
)
|
||||
for chunk_id, h in enumerate(group_hashes):
|
||||
start_idx = chunk_id * spec_block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
for tp in range(tp_count):
|
||||
for pp in range(self.pp_size):
|
||||
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)
|
||||
candidate_keys.append(PoolKey(md, h.hex()).to_string())
|
||||
candidate_meta.append((g_idx, bytes(h)))
|
||||
|
||||
if not candidate_keys:
|
||||
return 0
|
||||
|
||||
try:
|
||||
starts: list[int] = []
|
||||
for start, end, key in self.token_database.process_tokens(
|
||||
token_len, block_hashes
|
||||
):
|
||||
keys.append(key.to_string())
|
||||
starts.append(start)
|
||||
|
||||
# Expand keys for all TP ranks
|
||||
multi_tp_keys = keys[:]
|
||||
for i in range(1, min(self.tp_size, self.num_kv_head)):
|
||||
for item in keys:
|
||||
new_str = item.replace("@tp_rank:0", f"@tp_rank:{i}", 1)
|
||||
multi_tp_keys.append(new_str)
|
||||
|
||||
# Expand keys for all PP ranks
|
||||
pp_base_keys = multi_tp_keys.copy()
|
||||
for i in range(1, self.pp_size):
|
||||
for item in pp_base_keys:
|
||||
new_str = item.replace("@pp_rank:0", f"@pp_rank:{i}", 1)
|
||||
multi_tp_keys.append(new_str)
|
||||
|
||||
res = self.store.batch_is_exist(multi_tp_keys)
|
||||
|
||||
num_block = len(keys)
|
||||
multi_tp_values = [
|
||||
res[i * num_block : (i + 1) * num_block]
|
||||
for i in range(min(self.tp_size, self.num_kv_head) * self.pp_size)
|
||||
]
|
||||
index = self._find_min_first_non_one_index(multi_tp_values)
|
||||
if index != -1:
|
||||
return starts[index]
|
||||
res = self.store.batch_is_exist(candidate_keys)
|
||||
except Exception as e:
|
||||
logger.error("Remote connection failed in lookup: %s", e)
|
||||
return 0
|
||||
return end
|
||||
|
||||
@staticmethod
|
||||
def _find_min_first_non_one_index(
|
||||
arr: list[list[int]],
|
||||
) -> int:
|
||||
try:
|
||||
return min(idx for row in arr for idx, val in enumerate(row) if val != 1)
|
||||
except ValueError:
|
||||
return -1
|
||||
# A (group, hash) is "present" only when every TP*PP rank has it.
|
||||
expected_per_key = max(1, tp_count * self.pp_size)
|
||||
present_count: dict[tuple[int, bytes], int] = {}
|
||||
for gh, exists in zip(candidate_meta, res, strict=True):
|
||||
if exists == 1:
|
||||
present_count[gh] = present_count.get(gh, 0) + 1
|
||||
exists_set = {gh for gh, c in present_count.items() if c >= expected_per_key}
|
||||
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
|
||||
)
|
||||
return hit_length
|
||||
|
||||
def get_kv_events(self) -> list[BlockStored]:
|
||||
if self.enable_kv_events and self.kv_send_thread is not None:
|
||||
@@ -1220,7 +1276,8 @@ class LookupKeyServer:
|
||||
token_len = int.from_bytes(all_frames[0], byteorder="big")
|
||||
hash_frames = all_frames[1:]
|
||||
hashes_str = self.decoder.decode(hash_frames)
|
||||
result = self.store_worker.lookup(token_len, hashes_str)
|
||||
block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str]
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
response = result.to_bytes(4, "big")
|
||||
self.socket.send(response)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user