[KV Connector][Mooncake] Add cache_prefix to namespace store keys (#45767)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
188c68798e
commit
44b2512767
@@ -205,6 +205,7 @@ the vLLM JSON config.
|
||||
- `load_async` (bool): Enable asynchronous loading for better compute-I/O overlap. Default: `true`.
|
||||
- `enable_cross_layers_blocks` (bool): Enable cross-layer block packing for reduced store operations. Default: `false`.
|
||||
- `lookup_rpc_port` (int): Custom port for the ZMQ lookup RPC socket. Default: `0`.
|
||||
- `cache_prefix` (str): Namespace prepended to every store key. Lets separate deployments share one Mooncake master without polluting each other — instances configured with different prefixes never see each other's cached blocks, even for identical prompts. All instances that should share a prefix cache must use the same value. Default: `""` (no prefix; keys are byte-identical to the unprefixed format).
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
@@ -240,6 +241,31 @@ def _patch_worker_runtime(monkeypatch, *, local_ip: str = "10.0.0.7") -> None:
|
||||
monkeypatch.setattr(worker, "get_ip", lambda: local_ip)
|
||||
|
||||
|
||||
def test_pool_key_to_string_without_prefix_is_unchanged():
|
||||
"""Default (empty) cache_prefix keeps keys byte-identical to the
|
||||
historical unprefixed format so existing deployments keep their hits."""
|
||||
key = PoolKey(KeyMetadata("test-model", 0, 0, 0, 0), "deadbeef")
|
||||
assert (
|
||||
key.to_string() == "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
|
||||
)
|
||||
|
||||
|
||||
def test_pool_key_cache_prefix_namespaces_and_disambiguates():
|
||||
"""A non-empty cache_prefix is prepended, and two instances with
|
||||
different prefixes never collide on identical block hashes."""
|
||||
md_a = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depA")
|
||||
md_b = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depB")
|
||||
|
||||
key_a = PoolKey(md_a, "deadbeef")
|
||||
key_b = PoolKey(md_b, "deadbeef")
|
||||
|
||||
assert key_a.to_string() == (
|
||||
"depA@test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
|
||||
)
|
||||
assert key_a.to_string() != key_b.to_string()
|
||||
assert hash(key_a) != hash(key_b)
|
||||
|
||||
|
||||
def test_default_local_buffer_size_matches_pr40900():
|
||||
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
|
||||
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
|
||||
|
||||
@@ -33,6 +33,10 @@ class KeyMetadata:
|
||||
dcp_rank: int
|
||||
pp_rank: int
|
||||
group_id: int = 0
|
||||
# Optional namespace prepended to every key. Lets separate deployments
|
||||
# share one Mooncake master without colliding on identical block hashes.
|
||||
# Empty (the default) keeps keys byte-identical to the unprefixed format.
|
||||
cache_prefix: str = ""
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
@@ -45,6 +49,7 @@ class PoolKey:
|
||||
def __hash__(self):
|
||||
return hash(
|
||||
(
|
||||
self.key_metadata.cache_prefix,
|
||||
self.key_metadata.model_name,
|
||||
self.key_metadata.tp_rank,
|
||||
self.key_metadata.pcp_rank,
|
||||
@@ -56,7 +61,13 @@ class PoolKey:
|
||||
)
|
||||
|
||||
def to_string(self) -> str:
|
||||
prefix = (
|
||||
f"{self.key_metadata.cache_prefix}@"
|
||||
if self.key_metadata.cache_prefix
|
||||
else ""
|
||||
)
|
||||
return (
|
||||
f"{prefix}"
|
||||
f"{self.key_metadata.model_name}"
|
||||
f"@tp_rank:{self.key_metadata.tp_rank}"
|
||||
f"@pcp{self.key_metadata.pcp_rank}"
|
||||
|
||||
@@ -982,6 +982,11 @@ class MooncakeStoreWorker:
|
||||
pcp_rank=self.pcp_rank,
|
||||
dcp_rank=self.dcp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
cache_prefix=str(
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"cache_prefix", ""
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
|
||||
Reference in New Issue
Block a user