diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_prepare_values.py b/tests/v1/kv_connector/unit/test_mooncake_store_prepare_values.py new file mode 100644 index 0000000000..77cae42ffc --- /dev/null +++ b/tests/v1/kv_connector/unit/test_mooncake_store_prepare_values.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ChunkedTokenDatabase.prepare_values.""" + +import random + +import pytest + +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + ChunkedTokenDatabase, + KeyMetadata, +) +from vllm.utils.math_utils import cdiv + +BLOCK_SIZE = 128 + + +def _reference_prepare_value( + db: ChunkedTokenDatabase, start: int, end: int, block_ids: list[int] +) -> tuple[list[int], list[int], int]: + """Compute a token range with the original scalar implementation.""" + addr_list = [] + size_list = [] + block_id = block_ids[start // db.block_size] + length = len(db.block_len) + for index, base_addr in enumerate(db.kv_caches_base_addr): + addr = base_addr + block_id * db.block_len[index % length] + assert (end - start) % db.block_size == 0 + size = db.block_len[index % length] * cdiv(end - start, db.block_size) + addr_list.append(addr) + size_list.append(size) + return addr_list, size_list, block_id + + +def _make_db(num_regions: int, num_block_lens: int) -> ChunkedTokenDatabase: + md = KeyMetadata(model_name="t", tp_rank=1, pcp_rank=0, dcp_rank=0, pp_rank=0) + db = ChunkedTokenDatabase(md, BLOCK_SIZE) + db.set_kv_caches_base_addr( + [0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)] + ) + # Exercise repeated block lengths when there are more cache regions. + db.set_block_len([30_208 + 512 * i for i in range(num_block_lens)]) + return db + + +@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)]) +def test_prepare_values_matches_reference(num_regions: int, num_block_lens: int): + db = _make_db(num_regions, num_block_lens) + rng = random.Random(0) + n_blocks = 300 + block_ids = [rng.randrange(0, 1 << 20) for _ in range(n_blocks)] + chunks = [] + b = 0 + while b < n_blocks - 4: + span = rng.choice([1, 1, 1, 2, 4]) + chunks.append((b * BLOCK_SIZE, (b + span) * BLOCK_SIZE)) + b += span + rng.choice([0, 1]) + + addrs, sizes, bids = db.prepare_values(chunks, block_ids) + assert len(addrs) == len(sizes) == len(bids) == len(chunks) + for (start, end), addr, size, bid in zip(chunks, addrs, sizes, bids): + ref_addr, ref_size, ref_bid = _reference_prepare_value( + db, start, end, block_ids + ) + assert addr == ref_addr + assert size == ref_size + assert bid == ref_bid + # Native bindings require Python ints rather than numpy scalars. + assert all(type(a) is int for a in addr) + assert type(bid) is int + + +def test_prepare_value_single_matches_reference(): + db = _make_db(8, 8) + block_ids = list(range(64)) + got = db.prepare_value(5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids) + assert got == _reference_prepare_value( + db, 5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids + ) + + +def test_prepare_values_empty(): + db = _make_db(4, 4) + assert db.prepare_values([], [1, 2, 3]) == ([], [], []) + + +def test_prepare_values_rejects_unaligned_chunk(): + db = _make_db(4, 4) + with pytest.raises(AssertionError): + db.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1]) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index f8f43662f2..4d9e974b41 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -655,6 +655,68 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks(): assert masked_hashes == [b"a2".hex()] +def test_store_sending_thread_prepares_missing_chunks_once_per_group(): + store = MagicMock() + store.batch_is_exist.return_value = [0, 1, 0, 1, 0, 0] + store.batch_put_from_multi_buffers.return_value = [256, 256, 512, 512] + coord = SimpleNamespace( + lcm_block_size=16, + store_mask=lambda token_len, start_token, num_prompt_tokens=None: ( + None, + None, + ), + ) + + db0 = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=16, + ) + db0.set_kv_caches_base_addr([0x1000]) + db0.set_block_len([256]) + db0.prepare_values = MagicMock(wraps=db0.prepare_values) + db0.prepare_value = MagicMock(side_effect=AssertionError("scalar path called")) + + db1 = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), + block_size=16, + ) + db1.set_kv_caches_base_addr([0x2000]) + db1.set_block_len([512]) + db1.prepare_values = MagicMock(wraps=db1.prepare_values) + db1.prepare_value = MagicMock(side_effect=AssertionError("scalar path called")) + + thread = _make_store_sending_thread( + store, + coord=coord, + token_databases=[db0, db1], + ) + thread.add_stored_request("req-a") + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=48, + block_ids=([0, 1, 2], [2, 1, 0]), + block_hashes=[b"a0", b"a1", b"a2"], + can_save=True, + ) + ) + + db0.prepare_value.assert_not_called() + db1.prepare_value.assert_not_called() + db0.prepare_values.assert_called_once_with([(0, 16), (32, 48)], [0, 1, 2]) + db1.prepare_values.assert_called_once_with([(16, 32), (32, 48)], [2, 1, 0]) + + keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args + assert [key.rsplit("@", 1)[-1] for key in keys] == [ + "6130", + "6132", + "6131", + "6132", + ] + assert addrs == [[0x1000], [0x1200], [0x2200], [0x2000]] + assert sizes == [[256], [256], [512], [512]] + + def test_store_sending_thread_only_skips_on_no_available_handle(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index ef98ec0d4e..57e6bd8de0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -9,6 +9,7 @@ from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import cast +import numpy as np import torch from vllm.distributed.kv_transfer.kv_connector.v1.base import ( @@ -196,22 +197,46 @@ class ChunkedTokenDatabase: def prepare_value( self, start: int, end: int, block_ids: list[int] ) -> tuple[list[int], list[int], int]: - """Compute memory addresses and sizes for a token range. + """Compute memory addresses and sizes for a single token range. Returns: (addr_list, size_list, block_id) """ - addr_list = [] - size_list = [] - block_id = block_ids[start // self.block_size] + addr_lists, size_lists, chunk_block_ids = self.prepare_values( + ((start, end),), block_ids + ) + return addr_lists[0], size_lists[0], chunk_block_ids[0] + + def prepare_values( + self, + chunks: Sequence[tuple[int, int]], + block_ids: list[int], + ) -> tuple[list[list[int]], list[list[int]], list[int]]: + """Compute memory addresses and sizes for multiple token ranges. + + Returns: + (addr_lists, size_lists, chunk_block_ids), one entry per chunk. + """ + if not chunks: + return [], [], [] + base = np.asarray(self.kv_caches_base_addr, dtype=np.int64) length = len(self.block_len) - for index, base_addr in enumerate(self.kv_caches_base_addr): - addr = base_addr + block_id * self.block_len[index % length] - assert (end - start) % self.block_size == 0 - size = self.block_len[index % length] * cdiv(end - start, self.block_size) - addr_list.append(addr) - size_list.append(size) - return addr_list, size_list, block_id + blen = np.asarray( + [self.block_len[i % length] for i in range(base.shape[0])], + dtype=np.int64, + ) + n = len(chunks) + starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n) + spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts + assert not (spans % self.block_size).any() + bids = np.fromiter( + (block_ids[i] for i in (starts // self.block_size).tolist()), + dtype=np.int64, + count=n, + ) + addrs = base[None, :] + bids[:, None] * blen[None, :] + sizes = blen[None, :] * (spans // self.block_size)[:, None] + return addrs.tolist(), sizes.tolist(), bids.tolist() def process_tokens( self, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 38f4cb0c3a..0ca73d4211 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -634,6 +634,21 @@ class KVCacheStoreSendingThread(KVTransferThread): addrs: list[list[int]] = [] sizes: list[list[int]] = [] stored_events: list[BlockStored] = [] + chunks_per_group: list[list[tuple[int, int]]] = [ + [] for _ in self.token_databases + ] + for start, end, g_idx in zip(starts, ends, group_indices, strict=True): + chunks_per_group[g_idx].append((start, end)) + for g_idx, chunks in enumerate(chunks_per_group): + if not chunks: + continue + db = self.token_databases[g_idx] + group_addrs, group_sizes, _ = db.prepare_values( + chunks, block_ids_per_group[g_idx] + ) + addrs.extend(group_addrs) + sizes.extend(group_sizes) + # parent_block_hash chains live within a group, not across. if self.enable_kv_event: prev_key_per_group: dict[int, Any] = {} @@ -645,10 +660,6 @@ class KVCacheStoreSendingThread(KVTransferThread): zip(starts, ends, group_indices, strict=True) ): db = self.token_databases[g_idx] - addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx]) - addrs.append(addr) - sizes.append(size) - if self.enable_kv_event: token_ids = ( req_meta.token_ids[s:e] @@ -805,19 +816,21 @@ class KVCacheStoreRecvingThread(KVTransferThread): block_id_list: list[int] = [] for g_idx, db in enumerate(self.token_databases): mask = load_mask_per_group[g_idx] + chunks: list[tuple[int, int]] = [] for start, end, block_hash in db.process_tokens( token_len, req_meta.block_hashes, mask_num ): chunk_idx = start // db.block_size if chunk_idx >= len(mask) or not mask[chunk_idx]: continue - addr, size, block_id = db.prepare_value( - start, end, req_meta.block_ids[g_idx] - ) key_list.append(db.key_for(block_hash)) - addr_list.append(addr) - size_list.append(size) - block_id_list.append(block_id) + chunks.append((start, end)) + g_addrs, g_sizes, g_block_ids = db.prepare_values( + chunks, req_meta.block_ids[g_idx] + ) + addr_list.extend(g_addrs) + size_list.extend(g_sizes) + block_id_list.extend(g_block_ids) # Rotate aligned lists by tp_rank for load balancing. rotation = self.tp_rank % len(key_list)