[Perf][KVConnector][Mooncake] Vectorize prepare_value on the KV load path (#48531)

Signed-off-by: girasoley <girasoley@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: girasoley <girasoley@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Summer Yang
2026-07-23 18:12:00 +08:00
committed by GitHub
co-authored by Claude Fable 5 girasoley OpenAI Codex
parent f83de6d44c
commit a4904ba903
4 changed files with 211 additions and 21 deletions
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ChunkedTokenDatabase.prepare_values."""
import random
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
)
from vllm.utils.math_utils import cdiv
BLOCK_SIZE = 128
def _reference_prepare_value(
db: ChunkedTokenDatabase, start: int, end: int, block_ids: list[int]
) -> tuple[list[int], list[int], int]:
"""Compute a token range with the original scalar implementation."""
addr_list = []
size_list = []
block_id = block_ids[start // db.block_size]
length = len(db.block_len)
for index, base_addr in enumerate(db.kv_caches_base_addr):
addr = base_addr + block_id * db.block_len[index % length]
assert (end - start) % db.block_size == 0
size = db.block_len[index % length] * cdiv(end - start, db.block_size)
addr_list.append(addr)
size_list.append(size)
return addr_list, size_list, block_id
def _make_db(num_regions: int, num_block_lens: int) -> ChunkedTokenDatabase:
md = KeyMetadata(model_name="t", tp_rank=1, pcp_rank=0, dcp_rank=0, pp_rank=0)
db = ChunkedTokenDatabase(md, BLOCK_SIZE)
db.set_kv_caches_base_addr(
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
)
# Exercise repeated block lengths when there are more cache regions.
db.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
return db
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
def test_prepare_values_matches_reference(num_regions: int, num_block_lens: int):
db = _make_db(num_regions, num_block_lens)
rng = random.Random(0)
n_blocks = 300
block_ids = [rng.randrange(0, 1 << 20) for _ in range(n_blocks)]
chunks = []
b = 0
while b < n_blocks - 4:
span = rng.choice([1, 1, 1, 2, 4])
chunks.append((b * BLOCK_SIZE, (b + span) * BLOCK_SIZE))
b += span + rng.choice([0, 1])
addrs, sizes, bids = db.prepare_values(chunks, block_ids)
assert len(addrs) == len(sizes) == len(bids) == len(chunks)
for (start, end), addr, size, bid in zip(chunks, addrs, sizes, bids):
ref_addr, ref_size, ref_bid = _reference_prepare_value(
db, start, end, block_ids
)
assert addr == ref_addr
assert size == ref_size
assert bid == ref_bid
# Native bindings require Python ints rather than numpy scalars.
assert all(type(a) is int for a in addr)
assert type(bid) is int
def test_prepare_value_single_matches_reference():
db = _make_db(8, 8)
block_ids = list(range(64))
got = db.prepare_value(5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids)
assert got == _reference_prepare_value(
db, 5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids
)
def test_prepare_values_empty():
db = _make_db(4, 4)
assert db.prepare_values([], [1, 2, 3]) == ([], [], [])
def test_prepare_values_rejects_unaligned_chunk():
db = _make_db(4, 4)
with pytest.raises(AssertionError):
db.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1])
@@ -655,6 +655,68 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
assert masked_hashes == [b"a2".hex()]
def test_store_sending_thread_prepares_missing_chunks_once_per_group():
store = MagicMock()
store.batch_is_exist.return_value = [0, 1, 0, 1, 0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256, 512, 512]
coord = SimpleNamespace(
lcm_block_size=16,
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
None,
None,
),
)
db0 = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=16,
)
db0.set_kv_caches_base_addr([0x1000])
db0.set_block_len([256])
db0.prepare_values = MagicMock(wraps=db0.prepare_values)
db0.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
db1 = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
)
db1.set_kv_caches_base_addr([0x2000])
db1.set_block_len([512])
db1.prepare_values = MagicMock(wraps=db1.prepare_values)
db1.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db0, db1],
)
thread.add_stored_request("req-a")
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=48,
block_ids=([0, 1, 2], [2, 1, 0]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
)
)
db0.prepare_value.assert_not_called()
db1.prepare_value.assert_not_called()
db0.prepare_values.assert_called_once_with([(0, 16), (32, 48)], [0, 1, 2])
db1.prepare_values.assert_called_once_with([(16, 32), (32, 48)], [2, 1, 0])
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
assert [key.rsplit("@", 1)[-1] for key in keys] == [
"6130",
"6132",
"6131",
"6132",
]
assert addrs == [[0x1000], [0x1200], [0x2200], [0x2000]]
assert sizes == [[256], [256], [512], [512]]
def test_store_sending_thread_only_skips_on_no_available_handle():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)