[Perf][KVConnector][Mooncake] Vectorize prepare_value on the KV load path (#48531)
Signed-off-by: girasoley <girasoley@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: girasoley <girasoley@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5
girasoley
OpenAI Codex
parent
f83de6d44c
commit
a4904ba903
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ChunkedTokenDatabase.prepare_values."""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
|
||||
def _reference_prepare_value(
|
||||
db: ChunkedTokenDatabase, start: int, end: int, block_ids: list[int]
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Compute a token range with the original scalar implementation."""
|
||||
addr_list = []
|
||||
size_list = []
|
||||
block_id = block_ids[start // db.block_size]
|
||||
length = len(db.block_len)
|
||||
for index, base_addr in enumerate(db.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * db.block_len[index % length]
|
||||
assert (end - start) % db.block_size == 0
|
||||
size = db.block_len[index % length] * cdiv(end - start, db.block_size)
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
return addr_list, size_list, block_id
|
||||
|
||||
|
||||
def _make_db(num_regions: int, num_block_lens: int) -> ChunkedTokenDatabase:
|
||||
md = KeyMetadata(model_name="t", tp_rank=1, pcp_rank=0, dcp_rank=0, pp_rank=0)
|
||||
db = ChunkedTokenDatabase(md, BLOCK_SIZE)
|
||||
db.set_kv_caches_base_addr(
|
||||
[0x7F00_0000_0000 + i * (1 << 30) for i in range(num_regions)]
|
||||
)
|
||||
# Exercise repeated block lengths when there are more cache regions.
|
||||
db.set_block_len([30_208 + 512 * i for i in range(num_block_lens)])
|
||||
return db
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_regions,num_block_lens", [(96, 96), (96, 2), (1, 1)])
|
||||
def test_prepare_values_matches_reference(num_regions: int, num_block_lens: int):
|
||||
db = _make_db(num_regions, num_block_lens)
|
||||
rng = random.Random(0)
|
||||
n_blocks = 300
|
||||
block_ids = [rng.randrange(0, 1 << 20) for _ in range(n_blocks)]
|
||||
chunks = []
|
||||
b = 0
|
||||
while b < n_blocks - 4:
|
||||
span = rng.choice([1, 1, 1, 2, 4])
|
||||
chunks.append((b * BLOCK_SIZE, (b + span) * BLOCK_SIZE))
|
||||
b += span + rng.choice([0, 1])
|
||||
|
||||
addrs, sizes, bids = db.prepare_values(chunks, block_ids)
|
||||
assert len(addrs) == len(sizes) == len(bids) == len(chunks)
|
||||
for (start, end), addr, size, bid in zip(chunks, addrs, sizes, bids):
|
||||
ref_addr, ref_size, ref_bid = _reference_prepare_value(
|
||||
db, start, end, block_ids
|
||||
)
|
||||
assert addr == ref_addr
|
||||
assert size == ref_size
|
||||
assert bid == ref_bid
|
||||
# Native bindings require Python ints rather than numpy scalars.
|
||||
assert all(type(a) is int for a in addr)
|
||||
assert type(bid) is int
|
||||
|
||||
|
||||
def test_prepare_value_single_matches_reference():
|
||||
db = _make_db(8, 8)
|
||||
block_ids = list(range(64))
|
||||
got = db.prepare_value(5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids)
|
||||
assert got == _reference_prepare_value(
|
||||
db, 5 * BLOCK_SIZE, 7 * BLOCK_SIZE, block_ids
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_values_empty():
|
||||
db = _make_db(4, 4)
|
||||
assert db.prepare_values([], [1, 2, 3]) == ([], [], [])
|
||||
|
||||
|
||||
def test_prepare_values_rejects_unaligned_chunk():
|
||||
db = _make_db(4, 4)
|
||||
with pytest.raises(AssertionError):
|
||||
db.prepare_values([(0, BLOCK_SIZE + 1)], [0, 1])
|
||||
@@ -655,6 +655,68 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
|
||||
assert masked_hashes == [b"a2".hex()]
|
||||
|
||||
|
||||
def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 1, 0, 1, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256, 512, 512]
|
||||
coord = SimpleNamespace(
|
||||
lcm_block_size=16,
|
||||
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
|
||||
None,
|
||||
None,
|
||||
),
|
||||
)
|
||||
|
||||
db0 = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=16,
|
||||
)
|
||||
db0.set_kv_caches_base_addr([0x1000])
|
||||
db0.set_block_len([256])
|
||||
db0.prepare_values = MagicMock(wraps=db0.prepare_values)
|
||||
db0.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
|
||||
|
||||
db1 = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
)
|
||||
db1.set_kv_caches_base_addr([0x2000])
|
||||
db1.set_block_len([512])
|
||||
db1.prepare_values = MagicMock(wraps=db1.prepare_values)
|
||||
db1.prepare_value = MagicMock(side_effect=AssertionError("scalar path called"))
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db0, db1],
|
||||
)
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=48,
|
||||
block_ids=([0, 1, 2], [2, 1, 0]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
db0.prepare_value.assert_not_called()
|
||||
db1.prepare_value.assert_not_called()
|
||||
db0.prepare_values.assert_called_once_with([(0, 16), (32, 48)], [0, 1, 2])
|
||||
db1.prepare_values.assert_called_once_with([(16, 32), (32, 48)], [2, 1, 0])
|
||||
|
||||
keys, addrs, sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert [key.rsplit("@", 1)[-1] for key in keys] == [
|
||||
"6130",
|
||||
"6132",
|
||||
"6131",
|
||||
"6132",
|
||||
]
|
||||
assert addrs == [[0x1000], [0x1200], [0x2200], [0x2000]]
|
||||
assert sizes == [[256], [256], [512], [512]]
|
||||
|
||||
|
||||
def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
|
||||
@@ -9,6 +9,7 @@ from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -196,22 +197,46 @@ class ChunkedTokenDatabase:
|
||||
def prepare_value(
|
||||
self, start: int, end: int, block_ids: list[int]
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Compute memory addresses and sizes for a token range.
|
||||
"""Compute memory addresses and sizes for a single token range.
|
||||
|
||||
Returns:
|
||||
(addr_list, size_list, block_id)
|
||||
"""
|
||||
addr_list = []
|
||||
size_list = []
|
||||
block_id = block_ids[start // self.block_size]
|
||||
addr_lists, size_lists, chunk_block_ids = self.prepare_values(
|
||||
((start, end),), block_ids
|
||||
)
|
||||
return addr_lists[0], size_lists[0], chunk_block_ids[0]
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
"""Compute memory addresses and sizes for multiple token ranges.
|
||||
|
||||
Returns:
|
||||
(addr_lists, size_lists, chunk_block_ids), one entry per chunk.
|
||||
"""
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
|
||||
length = len(self.block_len)
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * self.block_len[index % length]
|
||||
assert (end - start) % self.block_size == 0
|
||||
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
return addr_list, size_list, block_id
|
||||
blen = np.asarray(
|
||||
[self.block_len[i % length] for i in range(base.shape[0])],
|
||||
dtype=np.int64,
|
||||
)
|
||||
n = len(chunks)
|
||||
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
|
||||
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
|
||||
assert not (spans % self.block_size).any()
|
||||
bids = np.fromiter(
|
||||
(block_ids[i] for i in (starts // self.block_size).tolist()),
|
||||
dtype=np.int64,
|
||||
count=n,
|
||||
)
|
||||
addrs = base[None, :] + bids[:, None] * blen[None, :]
|
||||
sizes = blen[None, :] * (spans // self.block_size)[:, None]
|
||||
return addrs.tolist(), sizes.tolist(), bids.tolist()
|
||||
|
||||
def process_tokens(
|
||||
self,
|
||||
|
||||
@@ -634,6 +634,21 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
chunks_per_group: list[list[tuple[int, int]]] = [
|
||||
[] for _ in self.token_databases
|
||||
]
|
||||
for start, end, g_idx in zip(starts, ends, group_indices, strict=True):
|
||||
chunks_per_group[g_idx].append((start, end))
|
||||
for g_idx, chunks in enumerate(chunks_per_group):
|
||||
if not chunks:
|
||||
continue
|
||||
db = self.token_databases[g_idx]
|
||||
group_addrs, group_sizes, _ = db.prepare_values(
|
||||
chunks, block_ids_per_group[g_idx]
|
||||
)
|
||||
addrs.extend(group_addrs)
|
||||
sizes.extend(group_sizes)
|
||||
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
if self.enable_kv_event:
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
@@ -645,10 +660,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
):
|
||||
db = self.token_databases[g_idx]
|
||||
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
|
||||
addrs.append(addr)
|
||||
sizes.append(size)
|
||||
|
||||
if self.enable_kv_event:
|
||||
token_ids = (
|
||||
req_meta.token_ids[s:e]
|
||||
@@ -805,19 +816,21 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
chunks: list[tuple[int, int]] = []
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
chunk_idx = start // db.block_size
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
addr, size, block_id = db.prepare_value(
|
||||
start, end, req_meta.block_ids[g_idx]
|
||||
)
|
||||
key_list.append(db.key_for(block_hash))
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
block_id_list.append(block_id)
|
||||
chunks.append((start, end))
|
||||
g_addrs, g_sizes, g_block_ids = db.prepare_values(
|
||||
chunks, req_meta.block_ids[g_idx]
|
||||
)
|
||||
addr_list.extend(g_addrs)
|
||||
size_list.extend(g_sizes)
|
||||
block_id_list.extend(g_block_ids)
|
||||
|
||||
# Rotate aligned lists by tp_rank for load balancing.
|
||||
rotation = self.tp_rank % len(key_list)
|
||||
|
||||
Reference in New Issue
Block a user