diff --git a/docs/features/mooncake_store_connector_usage.md b/docs/features/mooncake_store_connector_usage.md index f23acae10c..bab6941097 100644 --- a/docs/features/mooncake_store_connector_usage.md +++ b/docs/features/mooncake_store_connector_usage.md @@ -205,6 +205,7 @@ the vLLM JSON config. - `load_async` (bool): Enable asynchronous loading for better compute-I/O overlap. Default: `true`. - `enable_cross_layers_blocks` (bool): Enable cross-layer block packing for reduced store operations. Default: `false`. - `lookup_rpc_port` (int): Custom port for the ZMQ lookup RPC socket. Default: `0`. +- `cache_prefix` (str): Namespace prepended to every store key. Lets separate deployments share one Mooncake master without polluting each other — instances configured with different prefixes never see each other's cached blocks, even for identical prompts. All instances that should share a prefix cache must use the same value. Default: `""` (no prefix; keys are byte-identical to the unprefixed format). ## Notes diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 1130a7d6a7..78e0f3cacb 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( ChunkedTokenDatabase, KeyMetadata, LoadSpec, + PoolKey, ReqMeta, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( @@ -240,6 +241,31 @@ def _patch_worker_runtime(monkeypatch, *, local_ip: str = "10.0.0.7") -> None: monkeypatch.setattr(worker, "get_ip", lambda: local_ip) +def test_pool_key_to_string_without_prefix_is_unchanged(): + """Default (empty) cache_prefix keeps keys byte-identical to the + historical unprefixed format so existing deployments keep their hits.""" + key = PoolKey(KeyMetadata("test-model", 0, 0, 0, 0), "deadbeef") + assert ( + key.to_string() == "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef" + ) + + +def test_pool_key_cache_prefix_namespaces_and_disambiguates(): + """A non-empty cache_prefix is prepended, and two instances with + different prefixes never collide on identical block hashes.""" + md_a = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depA") + md_b = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depB") + + key_a = PoolKey(md_a, "deadbeef") + key_b = PoolKey(md_b, "deadbeef") + + assert key_a.to_string() == ( + "depA@test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef" + ) + assert key_a.to_string() != key_b.to_string() + assert hash(key_a) != hash(key_b) + + def test_default_local_buffer_size_matches_pr40900(): """PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode patch preserves it (and the JSON key) so unchanged PR-40900 configs work.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 0136a26067..55e2bd0633 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -33,6 +33,10 @@ class KeyMetadata: dcp_rank: int pp_rank: int group_id: int = 0 + # Optional namespace prepended to every key. Lets separate deployments + # share one Mooncake master without colliding on identical block hashes. + # Empty (the default) keeps keys byte-identical to the unprefixed format. + cache_prefix: str = "" @dataclass(order=True) @@ -45,6 +49,7 @@ class PoolKey: def __hash__(self): return hash( ( + self.key_metadata.cache_prefix, self.key_metadata.model_name, self.key_metadata.tp_rank, self.key_metadata.pcp_rank, @@ -56,7 +61,13 @@ class PoolKey: ) def to_string(self) -> str: + prefix = ( + f"{self.key_metadata.cache_prefix}@" + if self.key_metadata.cache_prefix + else "" + ) return ( + f"{prefix}" f"{self.key_metadata.model_name}" f"@tp_rank:{self.key_metadata.tp_rank}" f"@pcp{self.key_metadata.pcp_rank}" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 105762ccfc..4160d426fe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -982,6 +982,11 @@ class MooncakeStoreWorker: pcp_rank=self.pcp_rank, dcp_rank=self.dcp_rank, pp_rank=self.pp_rank, + cache_prefix=str( + vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "cache_prefix", "" + ) + ), ) # Initialize MooncakeDistributedStore with its own TransferEngine