[Perf][KVConnector][Mooncake] Vectorize prepare_value on the KV load path (#48531)

Signed-off-by: girasoley <girasoley@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: girasoley <girasoley@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Summer Yang
2026-07-23 18:12:00 +08:00
committed by GitHub
co-authored by Claude Fable 5 girasoley OpenAI Codex
parent f83de6d44c
commit a4904ba903
4 changed files with 211 additions and 21 deletions
@@ -9,6 +9,7 @@ from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from typing import cast
import numpy as np
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -196,22 +197,46 @@ class ChunkedTokenDatabase:
def prepare_value(
self, start: int, end: int, block_ids: list[int]
) -> tuple[list[int], list[int], int]:
"""Compute memory addresses and sizes for a token range.
"""Compute memory addresses and sizes for a single token range.
Returns:
(addr_list, size_list, block_id)
"""
addr_list = []
size_list = []
block_id = block_ids[start // self.block_size]
addr_lists, size_lists, chunk_block_ids = self.prepare_values(
((start, end),), block_ids
)
return addr_lists[0], size_lists[0], chunk_block_ids[0]
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
"""Compute memory addresses and sizes for multiple token ranges.
Returns:
(addr_lists, size_lists, chunk_block_ids), one entry per chunk.
"""
if not chunks:
return [], [], []
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
length = len(self.block_len)
for index, base_addr in enumerate(self.kv_caches_base_addr):
addr = base_addr + block_id * self.block_len[index % length]
assert (end - start) % self.block_size == 0
size = self.block_len[index % length] * cdiv(end - start, self.block_size)
addr_list.append(addr)
size_list.append(size)
return addr_list, size_list, block_id
blen = np.asarray(
[self.block_len[i % length] for i in range(base.shape[0])],
dtype=np.int64,
)
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert not (spans % self.block_size).any()
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
sizes = blen[None, :] * (spans // self.block_size)[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def process_tokens(
self,
@@ -634,6 +634,21 @@ class KVCacheStoreSendingThread(KVTransferThread):
addrs: list[list[int]] = []
sizes: list[list[int]] = []
stored_events: list[BlockStored] = []
chunks_per_group: list[list[tuple[int, int]]] = [
[] for _ in self.token_databases
]
for start, end, g_idx in zip(starts, ends, group_indices, strict=True):
chunks_per_group[g_idx].append((start, end))
for g_idx, chunks in enumerate(chunks_per_group):
if not chunks:
continue
db = self.token_databases[g_idx]
group_addrs, group_sizes, _ = db.prepare_values(
chunks, block_ids_per_group[g_idx]
)
addrs.extend(group_addrs)
sizes.extend(group_sizes)
# parent_block_hash chains live within a group, not across.
if self.enable_kv_event:
prev_key_per_group: dict[int, Any] = {}
@@ -645,10 +660,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
zip(starts, ends, group_indices, strict=True)
):
db = self.token_databases[g_idx]
addr, size, _ = db.prepare_value(s, e, block_ids_per_group[g_idx])
addrs.append(addr)
sizes.append(size)
if self.enable_kv_event:
token_ids = (
req_meta.token_ids[s:e]
@@ -805,19 +816,21 @@ class KVCacheStoreRecvingThread(KVTransferThread):
block_id_list: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = load_mask_per_group[g_idx]
chunks: list[tuple[int, int]] = []
for start, end, block_hash in db.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
chunk_idx = start // db.block_size
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
addr, size, block_id = db.prepare_value(
start, end, req_meta.block_ids[g_idx]
)
key_list.append(db.key_for(block_hash))
addr_list.append(addr)
size_list.append(size)
block_id_list.append(block_id)
chunks.append((start, end))
g_addrs, g_sizes, g_block_ids = db.prepare_values(
chunks, req_meta.block_ids[g_idx]
)
addr_list.extend(g_addrs)
size_list.extend(g_sizes)
block_id_list.extend(g_block_ids)
# Rotate aligned lists by tp_rank for load balancing.
rotation = self.tp_rank % len(key_list)