From ab7fcbdd5dbcb457c61f722e0a854de29491cf4d Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 20 Jun 2026 15:00:11 -0700 Subject: [PATCH] [Perf][KVConnector][Mooncake] Compact chunk-hash keys and zero-copy lookup wire format (#45969) --- .../unit/test_mooncake_store_coordinator.py | 7 +- .../unit/test_mooncake_store_hma_e2e.py | 13 ++- .../unit/test_mooncake_store_worker.py | 38 +++++++- .../v1/mooncake/store/coordinator.py | 24 ++--- .../kv_connector/v1/mooncake/store/data.py | 87 +++++++++++++++++-- .../v1/mooncake/store/protocol.py | 5 +- .../kv_connector/v1/mooncake/store/worker.py | 43 +++++---- 7 files changed, 164 insertions(+), 53 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py index 8d00345157..0cddd56a60 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py @@ -7,7 +7,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp ExternalCachedBlockPool, MooncakeStoreCoordinator, ) -from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + chunk_hashes_for_block_size, +) +from vllm.v1.core.kv_cache_utils import BlockHash from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheGroupSpec, @@ -182,7 +185,7 @@ def test_coordinator_group_block_size_double_hash(): ] coord = _make_coord(groups, hash_block_size=16) hs = _hashes(4) - big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32)) + big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32)) exists = {(0, bytes(h)) for h in hs} exists |= {(1, bytes(bh)) for bh in big_hashes} cmap = ExternalCachedBlockPool(exists) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index 01d4f4821e..9e9a57cdf7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -323,8 +323,8 @@ def test_recv_skips_swa_blocks_before_window(): def test_chunked_token_database_hash_block_size_smaller_than_block_size(): """DSv4-style: hash_block_size=4, group block_size=16 — process_tokens - must merge every 4 fine hashes into one chunk hash via - BlockHashListWithBlockSize.""" + keys each 16-token chunk by its last fine hash, keeping the Mooncake key + at one digest instead of concatenating all 4 fine hashes.""" md = KeyMetadata("m", 0, 0, 0, 0, group_id=3) db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4) db.set_kv_caches_base_addr([0]) @@ -335,8 +335,7 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size(): assert len(out) == 2 assert out[0][0] == 0 and out[0][1] == 16 assert out[1][0] == 16 and out[1][1] == 32 - # Each chunk's hash is the concatenation of 4 fine hashes. - expected0 = b"".join(fine_hashes[0:4]).hex() - expected1 = b"".join(fine_hashes[4:8]).hex() - assert out[0][2].chunk_hash == expected0 - assert out[1][2].chunk_hash == expected1 + # Each chunk's hash is its last (4th) fine hash, which already chains the + # prior three. + assert out[0][2].chunk_hash == fine_hashes[3].hex() + assert out[1][2].chunk_hash == fine_hashes[7].hex() diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 5213805115..96dd866bab 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( worker as mooncake_store_worker, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + BlobBlockHashes, ChunkedTokenDatabase, KeyMetadata, LoadSpec, @@ -32,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( MooncakeStoreConnectorStats, ) +from vllm.v1.core.kv_cache_utils import BlockHash def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator: @@ -1179,9 +1181,9 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata(): assert full_event.group_idx == 0 assert full_event.block_size == 32 assert full_event.token_ids == list(range(32)) - assert full_event.block_hashes == [ - maybe_convert_block_hash(BlockHash(b"".join(hs))) - ] + # block_size=32 over hash_block_size=8 (scale 4): the chunk is keyed by its + # last sub-hash, not the concatenation of all four. + assert full_event.block_hashes == [maybe_convert_block_hash(BlockHash(hs[3]))] assert swa_event.group_idx == 1 assert swa_event.block_size == 8 @@ -1749,3 +1751,33 @@ def test_store_worker_close_swallows_store_errors(): worker.close() assert worker.store is None + + +def test_blob_block_hashes_wire_roundtrip(): + """The lookup wire format sends a ``hash_len`` frame plus the raw hashes + concatenated back-to-back; the server rebuilds them through a zero-copy + ``BlobBlockHashes`` view over the frame buffer.""" + hashes = [BlockHash(bytes([i]) * 16) for i in range(5)] + hash_len = len(hashes[0]) + + # Client side (LookupKeyClient._lookup): flat payload frame. + blob = b"".join(hashes) + + # Server side (LookupKeyServer): view over the frame buffer (a memoryview), + # never materializing the full hash list upfront. + view = BlobBlockHashes(memoryview(blob), hash_len) + + assert len(view) == 5 + assert list(view) == hashes # default Sequence iter terminates via IndexError + assert [bytes(h) for h in view] == hashes + assert bytes(view[-1]) == hashes[-1] + assert [bytes(h) for h in view[1:3]] == hashes[1:3] + with pytest.raises(IndexError): + _ = view[5] + + +def test_blob_block_hashes_empty(): + """Empty lookups send hash_len=0 and an empty payload.""" + view = BlobBlockHashes(memoryview(b""), 0) + assert len(view) == 0 + assert list(view) == [] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index b1513e7269..89ffb56003 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -2,13 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """External-store cache-hit coordinator for MooncakeStoreConnector.""" +from collections.abc import Sequence from typing import cast +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( + chunk_hashes_for_block_size, +) from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_utils import ( BlockHash, - BlockHashList, - BlockHashListWithBlockSize, KVCacheBlock, ) from vllm.v1.core.single_type_kv_cache_manager import ( @@ -120,7 +122,7 @@ class MooncakeStoreCoordinator: def find_longest_cache_hit( self, - block_hashes: list[BlockHash], + block_hashes: Sequence[BlockHash], max_length: int, cached_block_pool: ExternalCachedBlockPool, *, @@ -147,7 +149,7 @@ class MooncakeStoreCoordinator: def load_mask( self, - block_hashes: list[BlockHash], + block_hashes: Sequence[BlockHash], token_len: int, ) -> tuple[list[bool], ...]: """Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s @@ -236,17 +238,15 @@ class MooncakeStoreCoordinator: return tuple(masks) def block_hashes_for_spec( - self, block_hashes: list[BlockHash], spec: KVCacheSpec - ) -> BlockHashList: - if spec.block_size == self.hash_block_size: - return block_hashes - return BlockHashListWithBlockSize( + self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec + ) -> Sequence[BlockHash]: + return chunk_hashes_for_block_size( block_hashes, self.hash_block_size, spec.block_size ) def _find_hit_blocks( self, - block_hashes: list[BlockHash], + block_hashes: Sequence[BlockHash], max_length: int, cached_block_pool: ExternalCachedBlockPool, *, @@ -264,7 +264,7 @@ class MooncakeStoreCoordinator: spec, group_ids, manager_cls = self.attention_groups[0] hashes = self.block_hashes_for_spec(block_hashes, spec) hit_blocks = manager_cls.find_longest_cache_hit( - block_hashes=hashes, + block_hashes=hashes, # type: ignore[arg-type] max_length=max_length, kv_cache_group_ids=group_ids, block_pool=cast(BlockPool, cached_block_pool), @@ -304,7 +304,7 @@ class MooncakeStoreCoordinator: _max_length = min(curr_hit_length + spec.block_size, max_length) hashes = self.block_hashes_for_spec(block_hashes, spec) hit_blocks = manager_cls.find_longest_cache_hit( - block_hashes=hashes, + block_hashes=hashes, # type: ignore[arg-type] max_length=_max_length, kv_cache_group_ids=group_ids, block_pool=cast(BlockPool, cached_block_pool), diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 55e2bd0633..12ad46a848 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -5,8 +5,9 @@ # (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/). """Data classes for MooncakeStoreConnector.""" -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from dataclasses import dataclass +from typing import cast import torch @@ -23,6 +24,77 @@ from vllm.v1.core.kv_cache_utils import ( logger = init_logger(__name__) +class BlobBlockHashes(Sequence[BlockHash]): + """Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead + of materializing all hashes upfront. + """ + + def __init__(self, blob: memoryview, hash_len: int): + self._blob = blob + self._hash_len = hash_len + self._n = len(blob) // hash_len if hash_len else 0 + + def __len__(self) -> int: + return self._n + + def __getitem__(self, idx): + if isinstance(idx, slice): + return [self[i] for i in range(*idx.indices(self._n))] + if idx < 0: + idx += self._n + if not 0 <= idx < self._n: + raise IndexError(idx) + off = idx * self._hash_len + return BlockHash(self._blob[off : off + self._hash_len]) + + +class _CompactChunkHashList(BlockHashListWithBlockSize): + """View that keys each ``block_size`` chunk by the last constituent + ``hash_block_size`` hash instead of concatenating all of them. + + The engine chains block hashes (each hash folds in the previous one), so the + final sub-block hash of a chunk already uniquely identifies the whole chunk + and its prefix. Using it keeps a Mooncake key at a single hash digest + regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of + growing the key linearly with it (e.g. 64x for ``block_size=256``, + ``hash_block_size=4``). + """ + + def __init__( + self, + block_hashes: Sequence[BlockHash], + hash_block_size: int, + target_block_size: int, + ): + # Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not + # just ``list``; the base only indexes/sizes it. + assert target_block_size % hash_block_size == 0 + self.block_hashes = block_hashes # type: ignore[assignment] + self.scale_factor = target_block_size // hash_block_size + + def _get_value_at(self, idx: int) -> BlockHash: + return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1] + + +def chunk_hashes_for_block_size( + block_hashes: Sequence[BlockHash], + hash_block_size: int, + block_size: int, +) -> Sequence[BlockHash]: + """Map ``hash_block_size``-granular block hashes to one compact hash per + ``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes`` + unchanged when the two sizes are equal. + """ + if block_size == hash_block_size: + return block_hashes + # Structurally a Sequence[BlockHash] (indexable + sized); the base class + # just isn't declared as one. + return cast( + "Sequence[BlockHash]", + _CompactChunkHashList(block_hashes, hash_block_size, block_size), + ) + + @dataclass class KeyMetadata: """Metadata for constructing pool keys.""" @@ -138,18 +210,15 @@ class ChunkedTokenDatabase: Args: token_len: Total number of tokens. block_hashes: Block hashes computed at ``hash_block_size`` granularity. - When ``block_size > hash_block_size`` consecutive hashes are merged - up to the group's ``block_size`` via ``BlockHashListWithBlockSize``. + When ``block_size > hash_block_size`` each group's ``block_size`` chunk + is keyed by its last sub-hash via ``chunk_hashes_for_block_size``. mask_num: Number of tokens to skip from the beginning. """ if not block_hashes: return - if self.block_size == self.hash_block_size: - chunk_hashes: Iterable[BlockHash] = block_hashes - else: - chunk_hashes = BlockHashListWithBlockSize( - block_hashes, self.hash_block_size, self.block_size - ) + chunk_hashes: Iterable[BlockHash] = chunk_hashes_for_block_size( + block_hashes, self.hash_block_size, self.block_size + ) for chunk_id, h in enumerate(chunk_hashes): start_idx = chunk_id * self.block_size if start_idx >= token_len: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py index 1317d78167..fc91b0aeeb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py @@ -11,7 +11,10 @@ Wire format (REQ/REP over IPC): msg_type == LOOKUP_MSG: frame 1: token_len (u32 big-endian, 4 bytes) - frame 2..n: msgpack-encoded list[str] of block-hash hex digests + frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each + fixed-size block hash (0 when there are no hashes) + frame 3: raw block hashes concatenated back-to-back (each hash_len + bytes); the server splits on hash_len Response: [hit_count: u32 big-endian, 4 bytes] msg_type == RESET_MSG: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 62c2d30c9c..e5db88ccbf 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -18,7 +18,7 @@ import socket import threading import time from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Sequence from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from typing import Any, Literal, TypeVar @@ -45,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp MooncakeStoreCoordinator, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 + BlobBlockHashes, ChunkedTokenDatabase, KeyMetadata, MooncakeStoreConnectorMetadata, @@ -65,7 +66,6 @@ from vllm.v1.core.kv_cache_utils import ( resolve_kv_cache_block_sizes, ) from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec -from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder from .metrics import MooncakeStoreConnectorStats @@ -1372,7 +1372,7 @@ class MooncakeStoreWorker: return finished_sending - def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int: + def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int: """Check how many prefix tokens exist in the store. Checks across all TP ranks and PP ranks. @@ -1392,6 +1392,11 @@ class MooncakeStoreWorker: group_hashes = self.coord.block_hashes_for_spec( block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec ) + metadata_templates = [ + dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp) + for tp in range(tp_count) + for pp in range(self.pp_size) + ] for chunk_id, h in enumerate(group_hashes): start_idx = chunk_id * spec_block_size if start_idx >= token_len: @@ -1400,11 +1405,11 @@ class MooncakeStoreWorker: chunk_id >= len(lookup_mask) or not lookup_mask[chunk_id] ): continue - for tp in range(tp_count): - for pp in range(self.pp_size): - md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp) - candidate_keys.append(PoolKey(md, h.hex()).to_string()) - candidate_meta.append((g_idx, bytes(h))) + h_hex = h.hex() + h_bytes = bytes(h) + for md in metadata_templates: + candidate_keys.append(PoolKey(md, h_hex).to_string()) + candidate_meta.append((g_idx, h_bytes)) if not candidate_keys: return 0 @@ -1483,7 +1488,6 @@ class LookupKeyServer: store_worker: MooncakeStoreWorker, vllm_config: VllmConfig, ): - self.decoder = MsgpackDecoder() self.ctx = zmq.Context() # type: ignore[attr-defined] socket_path = get_zmq_rpc_path_lookup(vllm_config) self._ipc_path = socket_path.removeprefix("ipc://") @@ -1506,9 +1510,9 @@ class LookupKeyServer: if msg_type == LOOKUP_MSG: token_len = int.from_bytes(all_frames[1], byteorder="big") - hash_frames = all_frames[2:] - hashes_str = self.decoder.decode(hash_frames) - block_hashes = [BlockHash(bytes.fromhex(s)) for s in hashes_str] + hash_len = int.from_bytes(all_frames[2], byteorder="big") + blob = all_frames[3].buffer + block_hashes = BlobBlockHashes(blob, hash_len) result = self.store_worker.lookup(token_len, block_hashes) self.socket.send(result.to_bytes(4, "big")) @@ -1557,7 +1561,6 @@ class LookupKeyClient: """ def __init__(self, vllm_config: VllmConfig): - self.encoder = MsgpackEncoder() self.ctx = zmq.Context() # type: ignore[attr-defined] socket_path = get_zmq_rpc_path_lookup(vllm_config) self.socket = make_zmq_socket( @@ -1574,14 +1577,16 @@ class LookupKeyClient: self.futures: dict[str, Future[int]] = {} def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int: - hash_strs = [h.hex() for h in block_hashes] - hash_frames = self.encoder.encode(hash_strs) - token_len_bytes = token_len.to_bytes(4, byteorder="big") - all_frames = [LOOKUP_MSG, token_len_bytes] + list(hash_frames) + hash_len = len(block_hashes[0]) if block_hashes else 0 + all_frames = ( + LOOKUP_MSG, + token_len.to_bytes(4, byteorder="big"), + hash_len.to_bytes(2, byteorder="big"), + b"".join(block_hashes), + ) self.socket.send_multipart(all_frames, copy=False) resp = self.socket.recv() - result = int.from_bytes(resp, "big") - return result + return int.from_bytes(resp, "big") def lookup( self,