[Perf][KVConnector][Mooncake] Compact chunk-hash keys and zero-copy lookup wire format (#45969)
This commit is contained in:
@@ -7,7 +7,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
|
||||
ExternalCachedBlockPool,
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, BlockHashListWithBlockSize
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
@@ -182,7 +185,7 @@ def test_coordinator_group_block_size_double_hash():
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
big_hashes = list(BlockHashListWithBlockSize(hs, 16, 32))
|
||||
big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32))
|
||||
exists = {(0, bytes(h)) for h in hs}
|
||||
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
|
||||
@@ -323,8 +323,8 @@ def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
|
||||
must merge every 4 fine hashes into one chunk hash via
|
||||
BlockHashListWithBlockSize."""
|
||||
keys each 16-token chunk by its last fine hash, keeping the Mooncake key
|
||||
at one digest instead of concatenating all 4 fine hashes."""
|
||||
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
|
||||
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
|
||||
db.set_kv_caches_base_addr([0])
|
||||
@@ -335,8 +335,7 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
assert len(out) == 2
|
||||
assert out[0][0] == 0 and out[0][1] == 16
|
||||
assert out[1][0] == 16 and out[1][1] == 32
|
||||
# Each chunk's hash is the concatenation of 4 fine hashes.
|
||||
expected0 = b"".join(fine_hashes[0:4]).hex()
|
||||
expected1 = b"".join(fine_hashes[4:8]).hex()
|
||||
assert out[0][2].chunk_hash == expected0
|
||||
assert out[1][2].chunk_hash == expected1
|
||||
# Each chunk's hash is its last (4th) fine hash, which already chains the
|
||||
# prior three.
|
||||
assert out[0][2].chunk_hash == fine_hashes[3].hex()
|
||||
assert out[1][2].chunk_hash == fine_hashes[7].hex()
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker as mooncake_store_worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
BlobBlockHashes,
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
@@ -32,6 +33,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
@@ -1179,9 +1181,9 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
assert full_event.group_idx == 0
|
||||
assert full_event.block_size == 32
|
||||
assert full_event.token_ids == list(range(32))
|
||||
assert full_event.block_hashes == [
|
||||
maybe_convert_block_hash(BlockHash(b"".join(hs)))
|
||||
]
|
||||
# block_size=32 over hash_block_size=8 (scale 4): the chunk is keyed by its
|
||||
# last sub-hash, not the concatenation of all four.
|
||||
assert full_event.block_hashes == [maybe_convert_block_hash(BlockHash(hs[3]))]
|
||||
|
||||
assert swa_event.group_idx == 1
|
||||
assert swa_event.block_size == 8
|
||||
@@ -1749,3 +1751,33 @@ def test_store_worker_close_swallows_store_errors():
|
||||
worker.close()
|
||||
|
||||
assert worker.store is None
|
||||
|
||||
|
||||
def test_blob_block_hashes_wire_roundtrip():
|
||||
"""The lookup wire format sends a ``hash_len`` frame plus the raw hashes
|
||||
concatenated back-to-back; the server rebuilds them through a zero-copy
|
||||
``BlobBlockHashes`` view over the frame buffer."""
|
||||
hashes = [BlockHash(bytes([i]) * 16) for i in range(5)]
|
||||
hash_len = len(hashes[0])
|
||||
|
||||
# Client side (LookupKeyClient._lookup): flat payload frame.
|
||||
blob = b"".join(hashes)
|
||||
|
||||
# Server side (LookupKeyServer): view over the frame buffer (a memoryview),
|
||||
# never materializing the full hash list upfront.
|
||||
view = BlobBlockHashes(memoryview(blob), hash_len)
|
||||
|
||||
assert len(view) == 5
|
||||
assert list(view) == hashes # default Sequence iter terminates via IndexError
|
||||
assert [bytes(h) for h in view] == hashes
|
||||
assert bytes(view[-1]) == hashes[-1]
|
||||
assert [bytes(h) for h in view[1:3]] == hashes[1:3]
|
||||
with pytest.raises(IndexError):
|
||||
_ = view[5]
|
||||
|
||||
|
||||
def test_blob_block_hashes_empty():
|
||||
"""Empty lookups send hash_len=0 and an empty payload."""
|
||||
view = BlobBlockHashes(memoryview(b""), 0)
|
||||
assert len(view) == 0
|
||||
assert list(view) == []
|
||||
|
||||
Reference in New Issue
Block a user