[KV Connector][Mooncake] Add tenant ID support to MooncakeStoreConnector (#48069)
Signed-off-by: lzw <2994587066@qq.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -61,6 +61,9 @@ Create a JSON configuration file (e.g., `mooncake_config.json`):
|
||||
large prefills do not exceed the owner's SSD-write budget. Set this together
|
||||
with the matching `--enable_offload=true` flag on `mooncake_master` and on
|
||||
the external `mooncake_client` (if any).
|
||||
- `tenant_id`: Optional Mooncake tenant namespace. Producers and consumers
|
||||
that should share store data must use the same tenant id. Default:
|
||||
`"default"`.
|
||||
|
||||
Set the config path via environment variable:
|
||||
|
||||
@@ -181,6 +184,26 @@ Mooncake environment variables (`MOONCAKE_OFFLOAD_FILE_STORAGE_PATH`,
|
||||
`MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES`, etc.). Those are independent of
|
||||
the vLLM JSON config.
|
||||
|
||||
### Tenant Isolation
|
||||
|
||||
Set `tenant_id` in the Mooncake JSON config when different vLLM deployments should use separate Mooncake tenant namespaces:
|
||||
|
||||
```json
|
||||
{
|
||||
"mode": "embedded",
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"master_server_address": "127.0.0.1:50051",
|
||||
"global_segment_size": "80GB",
|
||||
"local_buffer_size": "4GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "",
|
||||
"enable_offload": false,
|
||||
"tenant_id": "tenant-a"
|
||||
}
|
||||
```
|
||||
|
||||
Strict isolation requires a Mooncake master started with `--enable_multi_tenants=true` and a tenant quota policy that registers each tenant. Non-default `tenant_id` also requires a Mooncake version whose `MooncakeDistributedStore.setup()` accepts the `tenant_id` parameter. In `standalone-store` mode, start the external `mooncake_client` with the matching tenant id because that process owns the real store client.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|
||||
@@ -1599,7 +1599,7 @@ def test_requester_worker_init_prefers_local_hostname_override(
|
||||
},
|
||||
),
|
||||
)
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(), _make_kv_cache_config())
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(rank=1), _make_kv_cache_config())
|
||||
|
||||
assert store.setup.call_args.args[0] == "worker-a:50053"
|
||||
|
||||
@@ -2857,6 +2857,7 @@ def test_config_defaults_to_embedded():
|
||||
assert cfg.global_segment_size == worker.DEFAULT_GLOBAL_SEGMENT_SIZE
|
||||
assert cfg.local_buffer_size == worker.DEFAULT_LOCAL_BUFFER_SIZE
|
||||
assert cfg.enable_offload is False
|
||||
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
|
||||
|
||||
|
||||
def test_config_pr40900_unchanged(tmp_path):
|
||||
@@ -2878,6 +2879,51 @@ def test_config_pr40900_unchanged(tmp_path):
|
||||
assert cfg.global_segment_size == 4 * 1024**3
|
||||
assert cfg.local_buffer_size == 4 * 1024**3
|
||||
assert cfg.enable_offload is False
|
||||
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
|
||||
|
||||
|
||||
def test_config_from_file_normalizes_tenant_id(tmp_path):
|
||||
config_path = _write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"tenant_id": " tenant-a ",
|
||||
},
|
||||
)
|
||||
|
||||
cfg = worker.MooncakeStoreConfig.from_file(config_path)
|
||||
|
||||
assert cfg.tenant_id == "tenant-a"
|
||||
|
||||
|
||||
def test_config_from_file_normalizes_empty_tenant_id_to_default(tmp_path):
|
||||
config_path = _write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"tenant_id": " ",
|
||||
},
|
||||
)
|
||||
|
||||
cfg = worker.MooncakeStoreConfig.from_file(config_path)
|
||||
|
||||
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
|
||||
|
||||
|
||||
def test_config_from_file_rejects_non_string_tenant_id(tmp_path):
|
||||
config_path = _write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"tenant_id": False,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError, match="tenant_id must be a string or null"):
|
||||
worker.MooncakeStoreConfig.from_file(config_path)
|
||||
|
||||
|
||||
def test_config_embedded_rejects_zero_segment():
|
||||
@@ -2961,6 +3007,7 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
assert store.setup.call_args.kwargs == {}
|
||||
# ReplicateConfig is built and carries the preferred_segment.
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
assert w.store_replicate_config.preferred_segment == "10.0.0.7:50053"
|
||||
@@ -3004,6 +3051,7 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
assert store.setup.call_args.kwargs == {}
|
||||
# No preferred_segment — ReplicateConfig is default-constructed (so the
|
||||
# preferred_segment field keeps its default value).
|
||||
assert w.preferred_segment is None
|
||||
@@ -3013,6 +3061,63 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
|
||||
def test_topology_forwards_non_default_tenant_id(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "4GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"tenant_id": " tenant-a ",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(rank=1), _make_kv_cache_config())
|
||||
|
||||
assert store.setup.call_args.kwargs == {"tenant_id": "tenant-a"}
|
||||
|
||||
|
||||
def test_non_default_tenant_preserves_setup_type_error(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
setup_error = TypeError(
|
||||
"setup(): incompatible function arguments; "
|
||||
"supported signature includes tenant_id"
|
||||
)
|
||||
store.setup.side_effect = setup_error
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "4GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"tenant_id": "tenant-a",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError) as exc_info:
|
||||
worker.MooncakeStoreWorker(_make_vllm_config(rank=1), _make_kv_cache_config())
|
||||
|
||||
assert exc_info.value is setup_error
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats/metrics tests (PR-35 port)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -80,6 +80,7 @@ logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
DEFAULT_TENANT_ID = "default"
|
||||
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE = -200
|
||||
_T = TypeVar("_T")
|
||||
@@ -134,6 +135,7 @@ class MooncakeStoreConfig:
|
||||
global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE
|
||||
local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE
|
||||
enable_offload: bool = False
|
||||
tenant_id: str = DEFAULT_TENANT_ID
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.mode not in ("embedded", "standalone-store"):
|
||||
@@ -162,6 +164,7 @@ class MooncakeStoreConfig:
|
||||
config.get("local_buffer_size", DEFAULT_LOCAL_BUFFER_SIZE)
|
||||
),
|
||||
enable_offload=bool(config.get("enable_offload", False)),
|
||||
tenant_id=_normalize_tenant_id(config.get("tenant_id", DEFAULT_TENANT_ID)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -174,6 +177,17 @@ class MooncakeStoreConfig:
|
||||
return MooncakeStoreConfig.from_file(config_path)
|
||||
|
||||
|
||||
def _normalize_tenant_id(value: Any) -> str:
|
||||
if value is None:
|
||||
return DEFAULT_TENANT_ID
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(
|
||||
f"tenant_id must be a string or null, got {type(value).__name__}: {value!r}"
|
||||
)
|
||||
tenant_id = value.strip()
|
||||
return tenant_id if tenant_id else DEFAULT_TENANT_ID
|
||||
|
||||
|
||||
def _parse_size(value: Any) -> int:
|
||||
"""Parse storage size strings with units: GB, MB, KB, B."""
|
||||
if isinstance(value, int):
|
||||
@@ -1245,6 +1259,9 @@ class MooncakeStoreWorker:
|
||||
self.store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
setup_kwargs: dict[str, str] = {}
|
||||
if store_config.tenant_id != DEFAULT_TENANT_ID:
|
||||
setup_kwargs["tenant_id"] = store_config.tenant_id
|
||||
ret = self.store.setup(
|
||||
local_hostname,
|
||||
store_config.metadata_server,
|
||||
@@ -1253,6 +1270,7 @@ class MooncakeStoreWorker:
|
||||
store_config.protocol,
|
||||
store_config.device_name,
|
||||
store_config.master_server_address,
|
||||
**setup_kwargs,
|
||||
)
|
||||
if ret != 0:
|
||||
msg = "Initialize MooncakeDistributedStore failed."
|
||||
@@ -1280,12 +1298,13 @@ class MooncakeStoreWorker:
|
||||
|
||||
logger.info(
|
||||
"Mooncake mode=%s (global_segment_size=%d, local_buffer_size=%d, "
|
||||
"preferred_segment=%s, enable_offload=%s)",
|
||||
"preferred_segment=%s, enable_offload=%s, tenant_id=%s)",
|
||||
store_config.mode,
|
||||
store_config.global_segment_size,
|
||||
store_config.local_buffer_size,
|
||||
preferred_segment or "<none>",
|
||||
store_config.enable_offload,
|
||||
store_config.tenant_id,
|
||||
)
|
||||
if store_config.mode == "embedded":
|
||||
if store_config.enable_offload and preferred_segment is None:
|
||||
|
||||
Reference in New Issue
Block a user