[KV Connector] Support heterogeneous TP sharing in Mooncake Store Connector (#53129)
Signed-off-by: Guanyi Chen <939416532@qq.com> Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn> Signed-off-by: Zheng Zhou <zhouzh93@mail2.sysu.edu.cn> Signed-off-by: zane <zhouzh93@mail2.sysu.edu.cn> Co-authored-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
@@ -48,6 +48,31 @@ def _make_vllm_config():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_config",
|
||||
[
|
||||
{"store_tp_size": 4},
|
||||
{"enable_store_tp_lcm": True, "prefill_tp_sizes": [6, 4]},
|
||||
{"prefill_tp_sizes": [4, 2]},
|
||||
{"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 0]},
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_store_tp_does_not_override_backend_kv_cache_layout(extra_config):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeStoreConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
|
||||
assert (
|
||||
mooncake_store_connector.MooncakeStoreConnector.get_required_kvcache_layout(
|
||||
vllm_config
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config() -> KVCacheConfig:
|
||||
"""Single-group full-attention KVCacheConfig — enough for the connector
|
||||
constructor's validate() pass."""
|
||||
@@ -90,15 +115,18 @@ def test_scheduler_requires_align_mode_for_mamba():
|
||||
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
def _make_block_stored() -> BlockStored:
|
||||
def _make_block_stored(
|
||||
block_hash: bytes = b"hash", group_idx: int | None = None
|
||||
) -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[b"hash"],
|
||||
block_hashes=[block_hash],
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3],
|
||||
block_size=16,
|
||||
lora_id=None,
|
||||
medium="cpu",
|
||||
lora_name=None,
|
||||
group_idx=group_idx,
|
||||
)
|
||||
|
||||
|
||||
@@ -169,7 +197,7 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
assert invalid_block_ids == {3, 4}
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
@@ -184,7 +212,7 @@ def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = []
|
||||
mock_worker_cls.return_value.enable_kv_events = False
|
||||
assert connector.get_kv_connector_kv_cache_events() is None
|
||||
|
||||
|
||||
@@ -248,6 +276,9 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
)
|
||||
|
||||
mock_worker_cls.return_value.get_kv_events.return_value = [event]
|
||||
mock_worker_cls.return_value.enable_kv_events = True
|
||||
mock_worker_cls.return_value.kv_send_thread = MagicMock()
|
||||
mock_worker_cls.return_value.group_tp_replication_factors = (1,)
|
||||
kv_events = connector.get_kv_connector_kv_cache_events()
|
||||
|
||||
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
|
||||
@@ -255,6 +286,119 @@ def test_get_kv_connector_kv_cache_events_wraps_worker_events():
|
||||
assert kv_events.get_all_events() == [event]
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_keeps_empty_worker_contribution():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.enable_kv_events = True
|
||||
worker.kv_send_thread = MagicMock()
|
||||
worker.group_tp_replication_factors = (2,)
|
||||
worker.get_kv_events.return_value = []
|
||||
|
||||
kv_events = connector.get_kv_connector_kv_cache_events()
|
||||
|
||||
assert isinstance(kv_events, mooncake_store_connector.MooncakeStoreKVEvents)
|
||||
assert kv_events.get_number_of_workers() == 1
|
||||
assert kv_events.get_all_events() == []
|
||||
|
||||
|
||||
def _aggregate_store_events(
|
||||
worker_events: list[list[BlockStored]],
|
||||
replication_factors: tuple[int, ...],
|
||||
) -> list[BlockStored]:
|
||||
combined = mooncake_store_connector.MooncakeStoreKVEvents(
|
||||
num_workers=1,
|
||||
group_tp_replication_factors=replication_factors,
|
||||
)
|
||||
combined.add_events(worker_events[0])
|
||||
for events in worker_events[1:]:
|
||||
combined.add_events(events)
|
||||
combined.increment_workers()
|
||||
combined.aggregate()
|
||||
return combined.get_all_events()
|
||||
|
||||
|
||||
def test_gqa_store_events_follow_tp_replica_striping():
|
||||
block_a = _make_block_stored(b"a", group_idx=0)
|
||||
block_b = _make_block_stored(b"b", group_idx=0)
|
||||
|
||||
events = _aggregate_store_events(
|
||||
[[block_a], [block_b], [block_a], [block_b]],
|
||||
replication_factors=(2,),
|
||||
)
|
||||
|
||||
assert events == [block_a, block_b]
|
||||
|
||||
|
||||
def test_mqa_store_events_require_one_replica():
|
||||
blocks = [_make_block_stored(bytes([index]), group_idx=0) for index in range(4)]
|
||||
|
||||
events = _aggregate_store_events(
|
||||
[[block] for block in blocks],
|
||||
replication_factors=(4,),
|
||||
)
|
||||
|
||||
assert events == blocks
|
||||
|
||||
|
||||
def test_store_events_require_every_non_replicated_worker():
|
||||
complete = _make_block_stored(b"complete", group_idx=0)
|
||||
incomplete = _make_block_stored(b"incomplete", group_idx=0)
|
||||
|
||||
events = _aggregate_store_events(
|
||||
[
|
||||
[complete, incomplete],
|
||||
[complete],
|
||||
[complete],
|
||||
[complete],
|
||||
],
|
||||
replication_factors=(1,),
|
||||
)
|
||||
|
||||
assert events == [complete]
|
||||
|
||||
|
||||
def test_store_events_reject_incomplete_replica_coverage():
|
||||
complete = _make_block_stored(b"complete", group_idx=0)
|
||||
incomplete = _make_block_stored(b"incomplete", group_idx=0)
|
||||
|
||||
events = _aggregate_store_events(
|
||||
[[complete, incomplete], [complete], [], []],
|
||||
replication_factors=(2,),
|
||||
)
|
||||
|
||||
assert events == [complete]
|
||||
|
||||
|
||||
def test_store_events_use_each_groups_replication_factor():
|
||||
full_attention = _make_block_stored(b"full", group_idx=0)
|
||||
replicated_attention = _make_block_stored(b"replicated", group_idx=1)
|
||||
|
||||
events = _aggregate_store_events(
|
||||
[
|
||||
[full_attention, replicated_attention],
|
||||
[full_attention],
|
||||
[full_attention, replicated_attention],
|
||||
[full_attention],
|
||||
],
|
||||
replication_factors=(1, 2),
|
||||
)
|
||||
|
||||
assert events == [full_attention, replicated_attention]
|
||||
|
||||
|
||||
def test_update_connector_output_and_take_events():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
@@ -284,6 +428,28 @@ def test_update_connector_output_and_take_events():
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
|
||||
def test_store_events_persist_across_polls():
|
||||
connector = object.__new__(mooncake_store_connector.MooncakeStoreConnector)
|
||||
connector.connector_scheduler = MagicMock()
|
||||
connector._kv_cache_events = None
|
||||
complete = _make_block_stored(b"complete", group_idx=0)
|
||||
pending = _make_block_stored(b"pending", group_idx=0)
|
||||
|
||||
def poll(worker_events: list[BlockStored]) -> list[BlockStored]:
|
||||
events = mooncake_store_connector.MooncakeStoreKVEvents(
|
||||
num_workers=4,
|
||||
group_tp_replication_factors=(2,),
|
||||
)
|
||||
events.add_events(worker_events)
|
||||
connector.update_connector_output(KVConnectorOutput(kv_cache_events=events))
|
||||
return list(connector.take_events())
|
||||
|
||||
assert poll([complete, pending]) == []
|
||||
assert poll([complete]) == [complete]
|
||||
assert poll([pending, pending]) == [pending]
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# reset_cache() — RL hard-reset path via typed LookupKey protocol
|
||||
# ============================================================
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Mooncake Store payload layouts."""
|
||||
|
||||
import ctypes
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LBHNCStoreLayout,
|
||||
LBNHCStoreLayout,
|
||||
RankLocalStoreLayout,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
|
||||
def _rank_local_layout(num_regions: int, num_block_lens: int) -> RankLocalStoreLayout:
|
||||
metadata = KeyMetadata("test-model", 1, 0, 0, 0)
|
||||
layout = RankLocalStoreLayout(metadata, BLOCK_SIZE, BLOCK_SIZE)
|
||||
layout.set_kv_caches_base_addr(
|
||||
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
|
||||
)
|
||||
layout.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
|
||||
return layout
|
||||
|
||||
|
||||
def _rank_local_reference(
|
||||
layout: RankLocalStoreLayout, start: int, end: int, block_ids: list[int]
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
block_id = block_ids[start // layout.block_size]
|
||||
length = len(layout.block_len)
|
||||
addrs = [
|
||||
base_addr + block_id * layout.block_len[index % length]
|
||||
for index, base_addr in enumerate(layout.kv_caches_base_addr)
|
||||
]
|
||||
sizes = [
|
||||
layout.block_len[index % length] * cdiv(end - start, layout.block_size)
|
||||
for index in range(len(layout.kv_caches_base_addr))
|
||||
]
|
||||
return addrs, sizes, block_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
|
||||
def test_rank_local_descriptors_match_reference(num_regions: int, num_block_lens: int):
|
||||
layout = _rank_local_layout(num_regions, num_block_lens)
|
||||
rng = random.Random(0)
|
||||
block_ids = [rng.randrange(0, 1 << 20) for _ in range(300)]
|
||||
chunks = []
|
||||
block = 0
|
||||
while block < len(block_ids) - 4:
|
||||
span = rng.choice([1, 1, 1, 2, 4])
|
||||
chunks.append((block * BLOCK_SIZE, (block + span) * BLOCK_SIZE))
|
||||
block += span + rng.choice([0, 1])
|
||||
|
||||
addrs, sizes, selected_blocks = layout.prepare_values(
|
||||
chunks, block_ids, [0] * len(chunks)
|
||||
)
|
||||
|
||||
for chunk, chunk_addrs, chunk_sizes, block_id in zip(
|
||||
chunks, addrs, sizes, selected_blocks, strict=True
|
||||
):
|
||||
assert (chunk_addrs, chunk_sizes, block_id) == _rank_local_reference(
|
||||
layout, *chunk, block_ids
|
||||
)
|
||||
assert all(type(addr) is int for addr in chunk_addrs)
|
||||
assert type(block_id) is int
|
||||
|
||||
|
||||
def test_rank_local_descriptors_handle_empty_and_invalid_chunks():
|
||||
layout = _rank_local_layout(4, 4)
|
||||
assert layout.prepare_values([], [1, 2, 3], []) == ([], [], [])
|
||||
with pytest.raises(AssertionError):
|
||||
layout.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1], [0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("layout_cls", "store_format"),
|
||||
[
|
||||
(LBHNCStoreLayout, "tp_shared_lbhnc"),
|
||||
(LBNHCStoreLayout, "tp_shared_lbnhc"),
|
||||
],
|
||||
)
|
||||
def test_tp_shared_layout_owns_store_namespace(layout_cls, store_format):
|
||||
assert (
|
||||
layout_cls.shared_namespace(4, 2)
|
||||
== f"@store_tp:4@store_pp:2@store_format:{store_format}"
|
||||
)
|
||||
|
||||
|
||||
def test_rank_local_database_api_rejects_tp_shared_layout():
|
||||
metadata = KeyMetadata("test-model", 0, 0, 0, 0)
|
||||
layout = LBHNCStoreLayout(
|
||||
metadata,
|
||||
block_size=16,
|
||||
hash_block_size=16,
|
||||
local_tp_size=2,
|
||||
store_tp_size=4,
|
||||
tp_rank=0,
|
||||
num_kv_heads=8,
|
||||
)
|
||||
database = ChunkedTokenDatabase(metadata, 16, store_layout=layout)
|
||||
|
||||
with pytest.raises(RuntimeError, match="rank-local"):
|
||||
database.prepare_value_for_block(0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("layout_cls", "producer_strides", "consumer_strides"),
|
||||
[
|
||||
(LBHNCStoreLayout, (128, 64, 4, 1), (256, 64, 4, 1)),
|
||||
(LBNHCStoreLayout, (128, 4, 8, 1), (256, 4, 16, 1)),
|
||||
],
|
||||
)
|
||||
def test_tp_shared_layout_round_trip_across_tp_sizes(
|
||||
layout_cls, producer_strides, consumer_strides
|
||||
):
|
||||
block_size = 16
|
||||
shape = (1, 2, block_size, 4)
|
||||
stored: dict[int, bytes] = {}
|
||||
producer_tensors = []
|
||||
|
||||
for tp_rank in range(4):
|
||||
tensor = torch.empty_strided(shape, producer_strides, dtype=torch.float16)
|
||||
tensor.copy_(
|
||||
torch.arange(128, dtype=torch.float16).view(shape) + tp_rank * 1000
|
||||
)
|
||||
producer_tensors.append(tensor)
|
||||
metadata = KeyMetadata("test-model", tp_rank, 0, 0, 0)
|
||||
layout = layout_cls(
|
||||
metadata,
|
||||
block_size,
|
||||
block_size,
|
||||
local_tp_size=4,
|
||||
store_tp_size=4,
|
||||
tp_rank=tp_rank,
|
||||
num_kv_heads=8,
|
||||
)
|
||||
layout.register_kv_caches([tensor], 1)
|
||||
addrs, sizes, _ = layout.prepare_values([(0, block_size)], [0], [tp_rank])
|
||||
stored[tp_rank] = b"".join(
|
||||
ctypes.string_at(addr, size)
|
||||
for addr, size in zip(addrs[0], sizes[0], strict=True)
|
||||
)
|
||||
|
||||
for tp_rank in range(2):
|
||||
tensor = torch.empty_strided(
|
||||
(1, 4, block_size, 4), consumer_strides, dtype=torch.float16
|
||||
)
|
||||
tensor.zero_()
|
||||
metadata = KeyMetadata("test-model", tp_rank, 0, 0, 0)
|
||||
layout = layout_cls(
|
||||
metadata,
|
||||
block_size,
|
||||
block_size,
|
||||
local_tp_size=2,
|
||||
store_tp_size=4,
|
||||
tp_rank=tp_rank,
|
||||
num_kv_heads=8,
|
||||
)
|
||||
layout.register_kv_caches([tensor], 1)
|
||||
shard_ids = layout.local_shard_ids
|
||||
addrs, sizes, _ = layout.prepare_values(
|
||||
[(0, block_size)] * len(shard_ids), [0], shard_ids
|
||||
)
|
||||
for shard_id, shard_addrs, shard_sizes in zip(
|
||||
shard_ids, addrs, sizes, strict=True
|
||||
):
|
||||
offset = 0
|
||||
for addr, size in zip(shard_addrs, shard_sizes, strict=True):
|
||||
ctypes.memmove(addr, stored[shard_id][offset : offset + size], size)
|
||||
offset += size
|
||||
|
||||
expected = torch.cat(producer_tensors[tp_rank * 2 : tp_rank * 2 + 2], dim=1)
|
||||
torch.testing.assert_close(tensor, expected)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ChunkedTokenDatabase.prepare_values."""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
|
||||
def _reference_prepare_value(
|
||||
db: ChunkedTokenDatabase, start: int, end: int, block_ids: list[int]
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Compute a token range with the original scalar implementation."""
|
||||
addr_list = []
|
||||
size_list = []
|
||||
block_id = block_ids[start // db.block_size]
|
||||
length = len(db.block_len)
|
||||
for index, base_addr in enumerate(db.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * db.block_len[index % length]
|
||||
assert (end - start) % db.block_size == 0
|
||||
size = db.block_len[index % length] * cdiv(end - start, db.block_size)
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
return addr_list, size_list, block_id
|
||||
|
||||
|
||||
def _make_db(num_regions: int, num_block_lens: int) -> ChunkedTokenDatabase:
|
||||
md = KeyMetadata(model_name="t", tp_rank=1, pcp_rank=0, dcp_rank=0, pp_rank=0)
|
||||
db = ChunkedTokenDatabase(md, BLOCK_SIZE)
|
||||
db.set_kv_caches_base_addr(
|
||||
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
|
||||
)
|
||||
# Exercise repeated block lengths when there are more cache regions.
|
||||
db.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
|
||||
return db
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
|
||||
def test_prepare_values_matches_reference(num_regions: int, num_block_lens: int):
|
||||
db = _make_db(num_regions, num_block_lens)
|
||||
rng = random.Random(0)
|
||||
n_blocks = 300
|
||||
block_ids = [rng.randrange(0, 1 << 20) for _ in range(n_blocks)]
|
||||
chunks = []
|
||||
b = 0
|
||||
while b < n_blocks - 4:
|
||||
span = rng.choice([1, 1, 1, 2, 4])
|
||||
chunks.append((b * BLOCK_SIZE, (b + span) * BLOCK_SIZE))
|
||||
b += span + rng.choice([0, 1])
|
||||
|
||||
addrs, sizes, bids = db.prepare_values(chunks, block_ids)
|
||||
assert len(addrs) == len(sizes) == len(bids) == len(chunks)
|
||||
for (start, end), addr, size, bid in zip(chunks, addrs, sizes, bids):
|
||||
ref_addr, ref_size, ref_bid = _reference_prepare_value(
|
||||
db, start, end, block_ids
|
||||
)
|
||||
assert addr == ref_addr
|
||||
assert size == ref_size
|
||||
assert bid == ref_bid
|
||||
# Native bindings require Python ints rather than numpy scalars.
|
||||
assert all(type(a) is int for a in addr)
|
||||
assert type(bid) is int
|
||||
|
||||
|
||||
def test_prepare_value_single_matches_reference():
|
||||
db = _make_db(8, 8)
|
||||
block_ids = list(range(64))
|
||||
got = db.prepare_value(5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids)
|
||||
assert got == _reference_prepare_value(
|
||||
db, 5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_values_empty():
|
||||
db = _make_db(4, 4)
|
||||
assert db.prepare_values([], [1, 2, 3]) == ([], [], [])
|
||||
|
||||
|
||||
def test_prepare_values_rejects_unaligned_chunk():
|
||||
db = _make_db(4, 4)
|
||||
with pytest.raises(AssertionError):
|
||||
db.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1])
|
||||
@@ -17,6 +17,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.distributed.kv_events import KVEventAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
|
||||
rdma_utils,
|
||||
)
|
||||
@@ -30,9 +31,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
BlobBlockHashes,
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LBHNCStoreLayout,
|
||||
LBNHCStoreLayout,
|
||||
LoadSpec,
|
||||
MooncakeLookupResult,
|
||||
PoolKey,
|
||||
RankLocalStoreLayout,
|
||||
ReqMeta,
|
||||
TailKeyBoundary,
|
||||
)
|
||||
@@ -44,8 +48,19 @@ from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
RSWASpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
_TP_SHARED_NAMESPACE = "@store_tp:4@store_pp:1@store_format:tp_shared_lbhnc"
|
||||
_TP_SHARED_LBNHC_NAMESPACE = "@store_tp:4@store_pp:1@store_format:tp_shared_lbnhc"
|
||||
|
||||
|
||||
def _tp_shared_prefix(tp_rank: int) -> str:
|
||||
return (
|
||||
f"test-model{_TP_SHARED_NAMESPACE}@tp_rank:{tp_rank}"
|
||||
"@pcp0@dcp0@pp_rank:0@group:0"
|
||||
)
|
||||
|
||||
|
||||
class _RecordingBlockHashes:
|
||||
@@ -78,6 +93,46 @@ def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
)
|
||||
|
||||
|
||||
def _make_tp_shared_db(
|
||||
tp_rank: int = 0,
|
||||
) -> tuple[ChunkedTokenDatabase, torch.Tensor]:
|
||||
num_blocks = 2
|
||||
local_heads = 4
|
||||
block_size = 16
|
||||
content_size = 4
|
||||
strides = (
|
||||
local_heads * block_size * content_size,
|
||||
block_size * content_size,
|
||||
content_size,
|
||||
1,
|
||||
)
|
||||
tensor = torch.empty_strided(
|
||||
(num_blocks, local_heads, block_size, content_size),
|
||||
strides,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
metadata = KeyMetadata(
|
||||
"test-model",
|
||||
tp_rank,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
store_namespace=_TP_SHARED_NAMESPACE,
|
||||
)
|
||||
layout = LBHNCStoreLayout(
|
||||
metadata,
|
||||
block_size,
|
||||
block_size,
|
||||
local_tp_size=2,
|
||||
store_tp_size=4,
|
||||
tp_rank=tp_rank,
|
||||
num_kv_heads=8,
|
||||
)
|
||||
db = ChunkedTokenDatabase(metadata, block_size, store_layout=layout)
|
||||
layout.register_kv_caches([tensor], num_blocks)
|
||||
return db, tensor
|
||||
|
||||
|
||||
def _make_store_sending_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
@@ -243,11 +298,15 @@ def _make_vllm_config(
|
||||
rank: int = 0,
|
||||
decode_context_parallel_size: int = 1,
|
||||
kv_role: str = "kv_both",
|
||||
pipeline_parallel_size: int = 1,
|
||||
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
|
||||
) -> SimpleNamespace:
|
||||
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
|
||||
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
|
||||
return SimpleNamespace(
|
||||
model_config=_FakeModelConfig(),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
rank=rank,
|
||||
data_parallel_index=0,
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
@@ -256,7 +315,7 @@ def _make_vllm_config(
|
||||
kv_transfer_config=_FakeKVTransferConfig(
|
||||
kv_role=kv_role, extra_config=extra_config
|
||||
),
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
|
||||
cache_config=cache_config,
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
@@ -334,6 +393,138 @@ def test_pool_key_to_string_without_prefix_is_unchanged():
|
||||
)
|
||||
|
||||
|
||||
def test_pool_key_store_tp_namespace():
|
||||
key = PoolKey(
|
||||
KeyMetadata(
|
||||
"test-model",
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
store_namespace=_TP_SHARED_NAMESPACE,
|
||||
),
|
||||
"deadbeef",
|
||||
)
|
||||
|
||||
assert key.to_string() == f"{_tp_shared_prefix(2)}@deadbeef"
|
||||
|
||||
|
||||
def test_tp_shared_sending_writes_each_store_shard():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [1, 1, 1, 1]
|
||||
db, _ = _make_tp_shared_db()
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
token_databases=[db],
|
||||
replicate_config=SimpleNamespace(),
|
||||
)
|
||||
req = _make_store_req("req", [b"h0", b"h1"])
|
||||
_run_store_req(thread, req)
|
||||
|
||||
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
f"{_tp_shared_prefix(0)}@6830",
|
||||
f"{_tp_shared_prefix(1)}@6830",
|
||||
f"{_tp_shared_prefix(0)}@6831",
|
||||
f"{_tp_shared_prefix(1)}@6831",
|
||||
]
|
||||
assert len(addrs) == len(sizes) == 4
|
||||
|
||||
|
||||
def test_tp_shared_kv_events_aggregate_once_per_logical_block():
|
||||
aggregator = KVEventAggregator(num_workers=2)
|
||||
|
||||
for tp_rank in range(2):
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
db, _ = _make_tp_shared_db(tp_rank)
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
token_databases=[db],
|
||||
tp_rank=tp_rank,
|
||||
replicate_config=SimpleNamespace(),
|
||||
)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(16))
|
||||
|
||||
events = thread.get_kv_events()
|
||||
assert len(events) == 1
|
||||
aggregator.add_events(events)
|
||||
|
||||
assert len(aggregator.get_common_events()) == 1
|
||||
|
||||
|
||||
def test_tp_shared_kv_event_waits_for_all_missing_shards():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, -1]
|
||||
db, _ = _make_tp_shared_db()
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
token_databases=[db],
|
||||
replicate_config=SimpleNamespace(),
|
||||
)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(16))
|
||||
|
||||
assert thread.get_kv_events() == []
|
||||
|
||||
|
||||
def test_tp_shared_kv_event_counts_existing_shards_as_satisfied():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [1, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256]
|
||||
db, _ = _make_tp_shared_db()
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
token_databases=[db],
|
||||
replicate_config=SimpleNamespace(),
|
||||
)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(16))
|
||||
|
||||
assert len(thread.get_kv_events()) == 1
|
||||
|
||||
|
||||
def test_tp_shared_receiving_reads_each_local_store_shard():
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [1, 1, 1, 1]
|
||||
db, _ = _make_tp_shared_db()
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["layer0"], spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
thread = mooncake_store_worker.KVCacheStoreRecvingThread(
|
||||
store=store,
|
||||
token_databases=[db],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
|
||||
thread._handle_request(_make_load_req("req", [b"h0", b"h1"], token_len=32))
|
||||
|
||||
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
f"{_tp_shared_prefix(0)}@6830",
|
||||
f"{_tp_shared_prefix(1)}@6830",
|
||||
f"{_tp_shared_prefix(0)}@6831",
|
||||
f"{_tp_shared_prefix(1)}@6831",
|
||||
]
|
||||
assert len(addrs) == len(sizes) == 4
|
||||
|
||||
|
||||
def test_pool_key_cache_prefix_namespaces_and_disambiguates():
|
||||
"""A non-empty cache_prefix is prepended, and two instances with
|
||||
different prefixes never collide on identical block hashes."""
|
||||
@@ -1155,8 +1346,7 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
)
|
||||
db0.set_kv_caches_base_addr([0x1000])
|
||||
db0.set_block_len([256])
|
||||
db0.prepare_values = MagicMock(wraps=db0.prepare_values)
|
||||
db0.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
|
||||
db0.store_layout.prepare_values = MagicMock(wraps=db0.store_layout.prepare_values)
|
||||
|
||||
db1 = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
@@ -1164,8 +1354,7 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
)
|
||||
db1.set_kv_caches_base_addr([0x2000])
|
||||
db1.set_block_len([512])
|
||||
db1.prepare_values = MagicMock(wraps=db1.prepare_values)
|
||||
db1.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
|
||||
db1.store_layout.prepare_values = MagicMock(wraps=db1.store_layout.prepare_values)
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
@@ -1183,10 +1372,12 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
),
|
||||
)
|
||||
|
||||
db0.prepare_value.assert_not_called()
|
||||
db1.prepare_value.assert_not_called()
|
||||
db0.prepare_values.assert_called_once_with([(0, 16), (32, 48)], [1, 2, 3])
|
||||
db1.prepare_values.assert_called_once_with([(16, 32), (32, 48)], [3, 2, 1])
|
||||
db0.store_layout.prepare_values.assert_called_once_with(
|
||||
[(0, 16), (32, 48)], [1, 2, 3], [0, 0]
|
||||
)
|
||||
db1.store_layout.prepare_values.assert_called_once_with(
|
||||
[(16, 32), (32, 48)], [3, 2, 1], [0, 0]
|
||||
)
|
||||
|
||||
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert [key.rsplit("@", 1)[-1] for key in keys] == [
|
||||
@@ -1836,8 +2027,10 @@ def test_recv_thread_splits_disk_offload_loads_by_budget():
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
base_addr = thread.token_databases[0].kv_caches_base_addr[0]
|
||||
block_len = thread.token_databases[0].block_len[0]
|
||||
layout = thread.token_databases[0].store_layout
|
||||
assert isinstance(layout, RankLocalStoreLayout)
|
||||
base_addr = layout.kv_caches_base_addr[0]
|
||||
block_len = layout.block_len[0]
|
||||
assert first_addrs == [[base_addr], [base_addr + block_len]]
|
||||
assert second_addrs == [[base_addr + 2 * block_len]]
|
||||
expected_size = block_len
|
||||
@@ -2004,6 +2197,294 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
|
||||
def test_worker_enables_lbhnc_store_tp_layout(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=2)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(extra_config={"store_tp_size": 4}),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert w.store_tp_size == 4
|
||||
assert w.token_dbs[0].metadata.store_namespace == _TP_SHARED_NAMESPACE
|
||||
assert isinstance(w.token_dbs[0].store_layout, LBHNCStoreLayout)
|
||||
assert w.token_dbs[0].store_layout.local_shard_ids == (0, 1)
|
||||
assert len(w._lookup_key_prefixes[0]) == 4
|
||||
|
||||
|
||||
def test_worker_enables_lbnhc_store_tp_layout(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=2)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(worker.logger, "warning_once") as warning_once:
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
extra_config={"store_tp_size": 4},
|
||||
kv_cache_layout=KVCacheLayout.LBNHC,
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
warning_once.assert_called_once()
|
||||
assert w.store_tp_size == 4
|
||||
assert w.token_dbs[0].metadata.store_namespace == _TP_SHARED_LBNHC_NAMESPACE
|
||||
assert isinstance(w.token_dbs[0].store_layout, LBNHCStoreLayout)
|
||||
assert w.token_dbs[0].store_layout.local_shard_ids == (0, 1)
|
||||
assert len(w._lookup_key_prefixes[0]) == 4
|
||||
|
||||
|
||||
def test_tp_sharded_layout_rejects_full_attention_subclasses():
|
||||
spec = RSWASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
rswa_window=32,
|
||||
)
|
||||
store_worker = object.__new__(worker.MooncakeStoreWorker)
|
||||
store_worker.pcp_size = 1
|
||||
store_worker.dcp_size = 1
|
||||
store_worker._kv_cache_groups = [SimpleNamespace(kv_cache_spec=spec)]
|
||||
|
||||
assert not store_worker._supports_tp_sharded_store_layout(LBHNCStoreLayout, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_config", "expected"),
|
||||
[
|
||||
({"store_tp_size": 4}, 4),
|
||||
(
|
||||
{
|
||||
"enable_store_tp_lcm": True,
|
||||
"prefill_tp_sizes": [3, 4, 6],
|
||||
},
|
||||
12,
|
||||
),
|
||||
(
|
||||
{
|
||||
"enable_store_tp_lcm": False,
|
||||
"prefill_tp_sizes": [4, 2],
|
||||
},
|
||||
None,
|
||||
),
|
||||
({"enable_store_tp_lcm": True, "prefill_tp_sizes": []}, None),
|
||||
({"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 0]}, None),
|
||||
({"enable_store_tp_lcm": True, "prefill_tp_sizes": [4, 2.0]}, None),
|
||||
({"store_tp_size": 0}, None),
|
||||
({"store_tp_size": -1}, None),
|
||||
({"store_tp_size": True}, None),
|
||||
({"store_tp_size": 4.0}, None),
|
||||
],
|
||||
)
|
||||
def test_resolve_store_tp_size(extra_config: dict[str, object], expected: int | None):
|
||||
assert worker.resolve_store_tp_size(extra_config) == expected
|
||||
|
||||
|
||||
def test_lcm_store_tp_gives_prefill_and_decode_common_namespace(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
lcm_config = {
|
||||
"enable_store_tp_lcm": True,
|
||||
"prefill_tp_sizes": [4, 2],
|
||||
}
|
||||
|
||||
workers = []
|
||||
topologies = [
|
||||
(4, 2, "kv_both", (2,)),
|
||||
(2, 1, "kv_both", (2, 3)),
|
||||
(2, 1, "kv_consumer", (2, 3)),
|
||||
]
|
||||
for tp_size, tp_rank, kv_role, expected_shards in topologies:
|
||||
_patch_worker_runtime(
|
||||
monkeypatch,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
extra_config = dict(lcm_config)
|
||||
if kv_role == "kv_consumer":
|
||||
extra_config["save_decode_cache"] = True
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(extra_config=extra_config, kv_role=kv_role),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
assert store_worker.store_tp_size == 4
|
||||
assert isinstance(store_worker.token_dbs[0].store_layout, LBHNCStoreLayout)
|
||||
assert store_worker.token_dbs[0].store_layout.local_shard_ids == expected_shards
|
||||
workers.append(store_worker)
|
||||
|
||||
shard_keys = [
|
||||
store_worker.token_dbs[0].store_layout.key_for(2, BlockHash(b"h"))
|
||||
for store_worker in workers
|
||||
]
|
||||
assert len(set(shard_keys)) == 1
|
||||
assert workers[-1].can_put
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tp_size", "prefill_tp_sizes"),
|
||||
[
|
||||
(3, [4, 2]),
|
||||
(2, [4, 3]),
|
||||
(2, []),
|
||||
(2, [4, 0]),
|
||||
],
|
||||
)
|
||||
def test_lcm_store_tp_falls_back_when_topology_is_incompatible(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
tp_size: int,
|
||||
prefill_tp_sizes: list[int],
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=tp_size)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
extra_config={
|
||||
"enable_store_tp_lcm": True,
|
||||
"prefill_tp_sizes": prefill_tp_sizes,
|
||||
}
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert store_worker.store_tp_size is None
|
||||
assert type(store_worker.token_dbs[0]) is ChunkedTokenDatabase
|
||||
assert "@store_format:rank_local_tp" in store_worker.token_dbs[0].key_for(
|
||||
BlockHash(b"h")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("store_tp_size", [1, 3, 6])
|
||||
def test_worker_keeps_rank_local_layout_for_unsupported_store_tp(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
store_tp_size: int,
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=2)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
extra_config={"store_tp_size": store_tp_size},
|
||||
kv_cache_layout=KVCacheLayout.LBNHC,
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert w.store_tp_size is None
|
||||
assert type(w.token_dbs[0]) is ChunkedTokenDatabase
|
||||
key = w.token_dbs[0].key_for(BlockHash(b"h"))
|
||||
assert "@store_tp:" not in key
|
||||
assert "@store_pp:1" in key
|
||||
assert "@store_format:rank_local_tp2_layout_LBNHC" in key
|
||||
|
||||
|
||||
def test_store_tp_opt_in_isolates_different_pp_sizes(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch, tp_size=2)
|
||||
monkeypatch.setattr(_FakeModelConfig, "get_total_num_kv_heads", lambda _self: 8)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
keys = []
|
||||
for pp_size in (1, 2):
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
extra_config={"store_tp_size": 4},
|
||||
pipeline_parallel_size=pp_size,
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
layout = w.token_dbs[0].store_layout
|
||||
keys.append(layout.key_for(layout.local_shard_ids[0], BlockHash(b"h")))
|
||||
|
||||
assert "@store_pp:1" in keys[0]
|
||||
assert "@store_pp:2" in keys[1]
|
||||
assert keys[0] != keys[1]
|
||||
|
||||
|
||||
def test_save_decode_cache_keeps_transfer_path_enabled(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -2185,6 +2666,58 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
|
||||
)
|
||||
|
||||
|
||||
def test_mqa_p4_to_d2_uses_shared_rank_zero_namespace(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
block_hash = BlockHash(b"hash")
|
||||
put_keys: set[str] = set()
|
||||
get_keys: set[str] = set()
|
||||
for tp_size, target in ((4, put_keys), (2, get_keys)):
|
||||
for tp_rank in range(tp_size):
|
||||
_patch_worker_runtime(monkeypatch, tp_rank=tp_rank, tp_size=tp_size)
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
rank=tp_rank,
|
||||
extra_config={"store_tp_size": 4},
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert w.store_tp_size is None
|
||||
assert w.token_dbs[0].metadata.tp_rank == 0
|
||||
key = w.token_dbs[0].key_for(block_hash)
|
||||
if tp_size == 4:
|
||||
put_step = w._group_tp_replication_factors[0]
|
||||
if tp_rank % put_step == 0:
|
||||
target.add(key)
|
||||
else:
|
||||
target.add(key)
|
||||
|
||||
assert (
|
||||
put_keys
|
||||
== get_keys
|
||||
== {
|
||||
(
|
||||
"test-model@store_pp:1@store_format:tp_shared_mqa"
|
||||
"@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@68617368"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_requester_worker_group_semantics_falls_back_without_group_ids(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
@@ -2396,9 +2929,11 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, *_args: ([256] * len(keys))
|
||||
)
|
||||
|
||||
def successful_put(keys, addrs, sizes, *_args):
|
||||
return [256] * len(keys)
|
||||
|
||||
store.batch_put_from_multi_buffers.side_effect = successful_put
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=8, head_size=64, dtype=None
|
||||
@@ -2754,6 +3289,7 @@ def _make_bare_worker(
|
||||
worker.recv_request_queue = queue.Queue()
|
||||
worker.finished_store_req = set()
|
||||
worker.tp_size = 1
|
||||
worker.store_tp_size = None
|
||||
worker.num_kv_head = 1
|
||||
worker.pp_size = 1
|
||||
# Minimal single-full-attention-group config so the coordinator-based
|
||||
@@ -2812,6 +3348,40 @@ def test_lookup_key_prefixes_cover_dcp_rank_namespaces():
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_key_prefixes_cover_store_tp_shards():
|
||||
worker = _make_bare_worker()
|
||||
worker.store_tp_size = 4
|
||||
metadata = KeyMetadata(
|
||||
"test-model",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
group_id=0,
|
||||
store_namespace=_TP_SHARED_NAMESPACE,
|
||||
)
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
metadata,
|
||||
block_size=16,
|
||||
store_layout=LBHNCStoreLayout(
|
||||
metadata,
|
||||
block_size=16,
|
||||
hash_block_size=16,
|
||||
local_tp_size=2,
|
||||
store_tp_size=4,
|
||||
tp_rank=0,
|
||||
num_kv_heads=8,
|
||||
),
|
||||
)
|
||||
]
|
||||
worker._init_lookup_key_prefixes()
|
||||
|
||||
assert worker._lookup_key_prefixes[0] == (
|
||||
*(_tp_shared_prefix(tp_rank) for tp_rank in range(4)),
|
||||
)
|
||||
|
||||
|
||||
def test_lookup_key_prefixes_cover_pcp_rank_namespaces():
|
||||
worker = _make_bare_worker()
|
||||
worker.tp_size = 4
|
||||
|
||||
Reference in New Issue
Block a user