[KV Connector] Support disk offloading in MooncakeStoreConnector (#42689)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Zhewen Li
Claude Opus 4.7
parent
787bc0d031
commit
36e74c9ea4
@@ -1,23 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
|
||||
rdma_utils,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
worker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
ReqMeta,
|
||||
)
|
||||
|
||||
|
||||
def _make_store_sending_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
replicate_config: object | None = None,
|
||||
) -> worker.KVCacheStoreSendingThread:
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
@@ -32,11 +45,55 @@ def _make_store_sending_thread(
|
||||
put_step=1,
|
||||
kv_role="kv_producer",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=replicate_config,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
return thread
|
||||
|
||||
|
||||
def _make_store_recving_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
) -> worker.KVCacheStoreRecvingThread:
|
||||
token_database = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0), block_size=16
|
||||
)
|
||||
token_database.set_kv_caches_base_addr([0x1000])
|
||||
token_database.set_block_len([256])
|
||||
thread = worker.KVCacheStoreRecvingThread(
|
||||
store=store,
|
||||
token_database=token_database,
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
ready_event=threading.Event(),
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
return thread
|
||||
|
||||
|
||||
def _make_load_req(
|
||||
req_id: str,
|
||||
block_hashes: list[bytes],
|
||||
*,
|
||||
token_len: int,
|
||||
vllm_cached_tokens: int = 0,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=token_len,
|
||||
block_ids=list(range(len(block_hashes))),
|
||||
block_hashes=block_hashes,
|
||||
load_spec=LoadSpec(
|
||||
vllm_cached_tokens=vllm_cached_tokens,
|
||||
kvpool_cached_tokens=token_len,
|
||||
can_load=True,
|
||||
token_len=token_len,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
@@ -48,6 +105,233 @@ def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
)
|
||||
|
||||
|
||||
_DISK_OFFLOAD_SINGLE_KEY_BYTES = worker._estimate_disk_offload_staging_bytes([256])
|
||||
_DISK_OFFLOAD_USABLE_BUDGET_RATIO = 0.9
|
||||
_DISK_OFFLOAD_BUDGET_FOR_THREE_KEYS = 4 * _DISK_OFFLOAD_SINGLE_KEY_BYTES
|
||||
_DISK_OFFLOAD_BUDGET_FOR_SPLIT = math.ceil(
|
||||
2 * _DISK_OFFLOAD_SINGLE_KEY_BYTES / _DISK_OFFLOAD_USABLE_BUDGET_RATIO
|
||||
) # Allows two 256-byte chunks but not the third.
|
||||
_DISK_OFFLOAD_BUDGET_TOO_SMALL = (
|
||||
_DISK_OFFLOAD_SINGLE_KEY_BYTES - 1
|
||||
) # Smaller than a single 256-byte chunk.
|
||||
|
||||
|
||||
class _FakeKVTransferConfig:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
kv_role: str = "kv_both",
|
||||
extra_config: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
self.kv_role = kv_role
|
||||
self.kv_connector_extra_config = extra_config or {}
|
||||
|
||||
def get_from_extra_config(self, key: str, default: object) -> object:
|
||||
return self.kv_connector_extra_config.get(key, default)
|
||||
|
||||
|
||||
class _FakeModelConfig:
|
||||
model = "test-model"
|
||||
use_mla = False
|
||||
|
||||
def get_num_layers(self, parallel_config) -> int:
|
||||
return 1
|
||||
|
||||
def get_total_num_kv_heads(self) -> int:
|
||||
return 1
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
*, extra_config: dict[str, object] | None = None
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_config=_FakeModelConfig(),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
rank=0,
|
||||
),
|
||||
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
)
|
||||
|
||||
|
||||
def _write_mooncake_config(tmp_path, config: dict[str, object]) -> str:
|
||||
config_path = tmp_path / "mooncake_config.json"
|
||||
config_path.write_text(json.dumps(config), encoding="utf-8")
|
||||
return str(config_path)
|
||||
|
||||
|
||||
def _install_fake_mooncake(monkeypatch, store_instance: MagicMock):
|
||||
class FakeReplicateConfig:
|
||||
def __init__(self) -> None:
|
||||
self.preferred_segment = ""
|
||||
|
||||
fake_store_module = types.ModuleType("mooncake.store")
|
||||
fake_store_module.MooncakeDistributedStore = lambda: store_instance # type: ignore[attr-defined]
|
||||
fake_store_module.ReplicateConfig = FakeReplicateConfig # type: ignore[attr-defined]
|
||||
fake_mooncake_module = types.ModuleType("mooncake")
|
||||
fake_mooncake_module.store = fake_store_module # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "mooncake", fake_mooncake_module)
|
||||
monkeypatch.setitem(sys.modules, "mooncake.store", fake_store_module)
|
||||
return FakeReplicateConfig
|
||||
|
||||
|
||||
def _patch_worker_runtime(monkeypatch, *, local_ip: str = "10.0.0.7") -> None:
|
||||
single_rank_group = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
monkeypatch.setattr(worker, "get_mooncake_dp_engine_index", lambda _: 0)
|
||||
monkeypatch.setattr(worker, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(worker, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(worker, "get_pcp_group", lambda: single_rank_group)
|
||||
monkeypatch.setattr(worker, "get_dcp_group", lambda: single_rank_group)
|
||||
monkeypatch.setattr(worker, "get_ip", lambda: local_ip)
|
||||
|
||||
|
||||
def test_default_local_buffer_size_matches_pr40900():
|
||||
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
|
||||
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
|
||||
assert worker.DEFAULT_LOCAL_BUFFER_SIZE == 4 * 1024**3
|
||||
|
||||
|
||||
def test_get_requester_local_hostname_prefers_override(monkeypatch):
|
||||
monkeypatch.setenv("MOONCAKE_REQUESTER_LOCAL_HOSTNAME", "worker-a:50053")
|
||||
|
||||
assert rdma_utils.get_requester_local_hostname("10.0.0.7") == "worker-a:50053"
|
||||
|
||||
|
||||
def test_get_configured_preferred_segment_returns_explicit_override():
|
||||
assert (
|
||||
rdma_utils.get_configured_preferred_segment(
|
||||
{"preferred_segment": "10.0.0.7:50053"}
|
||||
)
|
||||
== "10.0.0.7:50053"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_preferred_segment_prefers_explicit_over_env(monkeypatch):
|
||||
monkeypatch.setenv("MOONCAKE_PREFERRED_SEGMENT", "10.0.0.8:50053")
|
||||
|
||||
assert (
|
||||
rdma_utils.get_configured_preferred_segment(
|
||||
{"preferred_segment": "10.0.0.7:50053"}
|
||||
)
|
||||
== "10.0.0.7:50053"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_preferred_segment_returns_env_override(monkeypatch):
|
||||
monkeypatch.setenv("MOONCAKE_PREFERRED_SEGMENT", "10.0.0.8:50053")
|
||||
|
||||
assert rdma_utils.get_configured_preferred_segment({}) == "10.0.0.8:50053"
|
||||
|
||||
|
||||
def test_get_configured_preferred_segment_rejects_empty_override():
|
||||
with pytest.raises(ValueError, match="preferred_segment"):
|
||||
rdma_utils.get_configured_preferred_segment({"preferred_segment": " "})
|
||||
|
||||
|
||||
def test_get_configured_worker_rnic_prefers_explicit_device_name(monkeypatch):
|
||||
store_config = worker.MooncakeStoreConfig(
|
||||
metadata_server="",
|
||||
local_buffer_size=1,
|
||||
protocol="rdma",
|
||||
device_name="rocep139s0",
|
||||
master_server_address="",
|
||||
)
|
||||
|
||||
assert (
|
||||
rdma_utils.get_configured_worker_rnic(
|
||||
protocol=store_config.protocol,
|
||||
configured_device=store_config.device_name,
|
||||
)
|
||||
== "rocep139s0"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_worker_rnic_selects_device_from_explicit_csv(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rdma_utils,
|
||||
"get_current_physical_gpu_index",
|
||||
lambda: 1,
|
||||
)
|
||||
store_config = worker.MooncakeStoreConfig(
|
||||
metadata_server="",
|
||||
local_buffer_size=1,
|
||||
protocol="rdma",
|
||||
device_name="rocep139s0,rocep140s0",
|
||||
master_server_address="",
|
||||
)
|
||||
|
||||
assert (
|
||||
rdma_utils.get_configured_worker_rnic(
|
||||
protocol=store_config.protocol,
|
||||
configured_device=store_config.device_name,
|
||||
)
|
||||
== "rocep140s0"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_worker_rnic_warns_and_returns_empty_for_rdma_with_no_device(
|
||||
caplog, monkeypatch
|
||||
):
|
||||
"""No device configured + protocol=rdma → emit a clear warning and return ""
|
||||
so the C++ side handles auto-selection. There is no Python-side fallback."""
|
||||
monkeypatch.setattr(logging.getLogger("vllm"), "propagate", True)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = rdma_utils.get_configured_worker_rnic(
|
||||
protocol="rdma",
|
||||
configured_device="",
|
||||
)
|
||||
assert result == ""
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert any("No RDMA devices specified" in r.message for r in warnings), (
|
||||
f"expected fallback warning, got {[r.message for r in warnings]}"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_worker_rnic_silent_for_tcp_with_no_device(caplog, monkeypatch):
|
||||
"""protocol=tcp + no device → return "" silently (no RDMA, no warning)."""
|
||||
monkeypatch.setattr(logging.getLogger("vllm"), "propagate", True)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = rdma_utils.get_configured_worker_rnic(
|
||||
protocol="tcp",
|
||||
configured_device="",
|
||||
)
|
||||
assert result == ""
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert not any("RDMA" in r.message for r in warnings), (
|
||||
"did not expect RDMA warning for tcp protocol, got "
|
||||
f"{[r.message for r in warnings]}"
|
||||
)
|
||||
|
||||
|
||||
def test_get_configured_worker_rnic_rejects_short_explicit_csv(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rdma_utils,
|
||||
"get_current_physical_gpu_index",
|
||||
lambda: 2,
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not cover local GPU 2"):
|
||||
rdma_utils.get_configured_worker_rnic(
|
||||
protocol="rdma",
|
||||
configured_device="rocep139s0,rocep140s0",
|
||||
)
|
||||
|
||||
|
||||
class _ReplicaDesc:
|
||||
def __init__(self, tier: str):
|
||||
self.tier = tier
|
||||
|
||||
def is_memory_replica(self) -> bool:
|
||||
return self.tier == "memory"
|
||||
|
||||
def is_disk_replica(self) -> bool:
|
||||
return self.tier == "disk"
|
||||
|
||||
def is_local_disk_replica(self) -> bool:
|
||||
return self.tier == "disk"
|
||||
|
||||
|
||||
def test_store_sending_thread_skips_request_during_cpu_pressure():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -105,6 +389,372 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053")
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert len(call_args) == 4
|
||||
assert call_args[3] is replicate_config
|
||||
|
||||
|
||||
def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_segment(): # noqa: E501
|
||||
"""Without a preferred_segment the SendingThread still forwards a
|
||||
(default-constructed) ReplicateConfig so the C++ side always sees a
|
||||
well-defined config object."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
replicate_config = SimpleNamespace()
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert len(call_args) == 4
|
||||
assert call_args[3] is replicate_config
|
||||
|
||||
|
||||
def test_estimate_disk_offload_staging_bytes_sums_multi_segment_sizes():
|
||||
assert worker._estimate_disk_offload_staging_bytes([256, 512]) == 12288
|
||||
|
||||
|
||||
def test_recv_thread_uses_single_batch_when_no_disk_offload_budget(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_MOONCAKE_STORE_TIER_LOG", raising=False)
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, 256, 256]
|
||||
thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
keys, addrs, sizes = store.batch_get_into_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
]
|
||||
assert sizes == [[256], [256], [256]]
|
||||
store.batch_get_replica_desc.assert_not_called()
|
||||
|
||||
|
||||
def test_recv_thread_logs_tier_summary_when_enabled(monkeypatch, caplog_vllm):
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_STORE_TIER_LOG", "1")
|
||||
caplog_vllm.set_level(logging.INFO, logger=worker.logger.name)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, 256, -10]
|
||||
thread = _make_store_recving_thread(store, disk_offload_buffer_budget_bytes=None)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
expected_keys = [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
]
|
||||
store.batch_get_replica_desc.return_value = {
|
||||
expected_keys[0]: [_ReplicaDesc("memory")],
|
||||
expected_keys[1]: [_ReplicaDesc("disk")],
|
||||
expected_keys[2]: [],
|
||||
}
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_replica_desc.call_args.args == (expected_keys,)
|
||||
assert store.method_calls[0][0] == "batch_get_replica_desc"
|
||||
assert store.method_calls[1][0] == "batch_get_into_multi_buffers"
|
||||
|
||||
messages = [record.getMessage() for record in caplog_vllm.records]
|
||||
assert any(
|
||||
"Mooncake load tier summary" in message
|
||||
and "req_id=req-a" in message
|
||||
and "batch_keys=3" in message
|
||||
and "memory_keys=1" in message
|
||||
and "disk_keys=1" in message
|
||||
and "unknown_keys=1" in message
|
||||
and "success_keys=2" in message
|
||||
and "failed_keys=1" in message
|
||||
and "bytes_by_tier={'memory': 256, 'disk': 256, 'unknown': 0}" in message
|
||||
for message in messages
|
||||
)
|
||||
|
||||
|
||||
def test_recv_thread_uses_ratio_scaled_budget_for_first_pass_split():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.side_effect = [
|
||||
[256],
|
||||
[256],
|
||||
]
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
disk_offload_buffer_budget_bytes=2 * _DISK_OFFLOAD_SINGLE_KEY_BYTES,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1"],
|
||||
token_len=32,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 2
|
||||
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
|
||||
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
]
|
||||
|
||||
|
||||
def test_recv_thread_splits_disk_offload_loads_by_budget():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.side_effect = [
|
||||
[256, 256],
|
||||
[256],
|
||||
]
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_FOR_SPLIT,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 2
|
||||
|
||||
first_keys = store.batch_get_into_multi_buffers.call_args_list[0].args[0]
|
||||
second_keys = store.batch_get_into_multi_buffers.call_args_list[1].args[0]
|
||||
first_addrs = store.batch_get_into_multi_buffers.call_args_list[0].args[1]
|
||||
second_addrs = store.batch_get_into_multi_buffers.call_args_list[1].args[1]
|
||||
first_sizes = store.batch_get_into_multi_buffers.call_args_list[0].args[2]
|
||||
second_sizes = store.batch_get_into_multi_buffers.call_args_list[1].args[2]
|
||||
assert first_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
]
|
||||
assert second_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
]
|
||||
base_addr = thread.token_database.kv_caches_base_addr[0]
|
||||
block_len = thread.token_database.block_len[0]
|
||||
assert first_addrs == [[base_addr], [base_addr + block_len]]
|
||||
assert second_addrs == [[base_addr + 2 * block_len]]
|
||||
expected_size = block_len
|
||||
assert first_sizes == [[expected_size], [expected_size]]
|
||||
assert second_sizes == [[expected_size]]
|
||||
|
||||
|
||||
def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [-10, -10]
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_FOR_SPLIT,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
"""PR-36 removed the count-based split trigger; with budget for 3 keys,
|
||||
all three should be requested in a single call."""
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, 256, 256]
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_FOR_THREE_KEYS,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
assert store.batch_get_into_multi_buffers.call_args_list[0].args[0] == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@6132",
|
||||
]
|
||||
|
||||
|
||||
def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
store = MagicMock()
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0"],
|
||||
token_len=16,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 0
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": "4gb",
|
||||
"local_buffer_size": "64mb",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"enable_offload": True,
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
|
||||
assert not hasattr(w, "_isolate_offload_resources")
|
||||
assert store.setup.call_args.args == (
|
||||
"10.0.0.7",
|
||||
"http://metadata/endpoint",
|
||||
4 * 1024 * 1024 * 1024, # global_segment_size: "4gb" honored
|
||||
64 * 1024 * 1024,
|
||||
"rdma",
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
|
||||
|
||||
def test_requester_worker_init_prefers_local_hostname_override(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv("MOONCAKE_REQUESTER_LOCAL_HOSTNAME", "worker-a:50053")
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"local_buffer_size": "64mb",
|
||||
"protocol": "tcp",
|
||||
"device_name": "",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
|
||||
assert store.setup.call_args.args[0] == "worker-a:50053"
|
||||
|
||||
|
||||
def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""enable_offload=False zeroes out the disk budget so we don't generate
|
||||
redundant owner GET-RPCs."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"device_name": "",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"enable_offload": False,
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
|
||||
def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"device_name": "",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
extra_config={
|
||||
"preferred_segment": "10.0.0.7:50053",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
assert w.store_replicate_config.preferred_segment == "10.0.0.7:50053"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers for register_kv_caches tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -147,8 +797,10 @@ def _make_bare_worker(
|
||||
w.tp_rank = 0
|
||||
w.put_step = 1
|
||||
w.enable_kv_events = False
|
||||
w.disk_offload_buffer_budget_bytes = None
|
||||
w.kv_send_thread = None
|
||||
w.kv_recv_thread = None
|
||||
w.store_replicate_config = SimpleNamespace()
|
||||
return w
|
||||
|
||||
|
||||
@@ -298,3 +950,186 @@ def test_register_kv_caches_cross_layer_single_segment():
|
||||
|
||||
assert w2.kv_caches_base_addr == w.kv_caches_base_addr
|
||||
assert w2.block_len == w.block_len
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dual-mode (embedded / standalone-store) config validation tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_config(**overrides):
|
||||
"""Build a MooncakeStoreConfig with sensible defaults for validation tests.
|
||||
|
||||
Required dataclass fields are populated; callers override only the field
|
||||
under test.
|
||||
"""
|
||||
base = dict(
|
||||
metadata_server="http://metadata/endpoint",
|
||||
master_server_address="10.0.0.7:50051",
|
||||
protocol="rdma",
|
||||
device_name="mlx5_0",
|
||||
)
|
||||
base.update(overrides)
|
||||
return worker.MooncakeStoreConfig(**base)
|
||||
|
||||
|
||||
def test_config_defaults_to_embedded():
|
||||
"""A JSON without explicit mode parses as embedded with 4 GiB segment."""
|
||||
cfg = _make_config()
|
||||
assert cfg.mode == "embedded"
|
||||
assert cfg.global_segment_size == worker.DEFAULT_GLOBAL_SEGMENT_SIZE
|
||||
assert cfg.local_buffer_size == worker.DEFAULT_LOCAL_BUFFER_SIZE
|
||||
assert cfg.enable_offload is False
|
||||
|
||||
|
||||
def test_config_pr40900_unchanged(tmp_path):
|
||||
"""A literal PR-40900 config (no mode, no enable_offload, no preferred_segment)
|
||||
parses without raising and resolves to embedded mode."""
|
||||
config_path = _write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "4GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
)
|
||||
cfg = worker.MooncakeStoreConfig.from_file(config_path)
|
||||
assert cfg.mode == "embedded"
|
||||
assert cfg.global_segment_size == 4 * 1024**3
|
||||
assert cfg.local_buffer_size == 4 * 1024**3
|
||||
assert cfg.enable_offload is False
|
||||
|
||||
|
||||
def test_config_embedded_rejects_zero_segment():
|
||||
with pytest.raises(
|
||||
ValueError, match=r"embedded mode requires global_segment_size > 0"
|
||||
):
|
||||
_make_config(mode="embedded", global_segment_size=0)
|
||||
|
||||
|
||||
def test_config_standalone_store_rejects_nonzero_segment():
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=r"standalone-store mode requires global_segment_size == 0",
|
||||
):
|
||||
_make_config(mode="standalone-store", global_segment_size=4 * 1024**3)
|
||||
|
||||
|
||||
def test_config_standalone_store_accepts_zero_segment():
|
||||
cfg = _make_config(mode="standalone-store", global_segment_size=0)
|
||||
assert cfg.mode == "standalone-store"
|
||||
assert cfg.global_segment_size == 0
|
||||
|
||||
|
||||
def test_config_unknown_mode():
|
||||
with pytest.raises(ValueError, match=r"unknown Mooncake mode"):
|
||||
_make_config(mode="something-else")
|
||||
|
||||
|
||||
def test_config_zero_local_buffer():
|
||||
with pytest.raises(ValueError, match=r"local_buffer_size must be > 0"):
|
||||
_make_config(local_buffer_size=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end topology tests
|
||||
# Covers the two supported recipes:
|
||||
# (A) standalone-store mode + disk offload (mode="standalone-store",
|
||||
# segment=0, enable_offload=true, preferred_segment set)
|
||||
# (B) embedded mode + CPU only (mode default, segment>0,
|
||||
# enable_offload=false, no preferred_segment)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
"""standalone-store + disk: global_segment_size=0, enable_offload=True,
|
||||
preferred_segment set. Assert setup() positional args, ReplicateConfig
|
||||
wiring, and that the disk-offload buffer budget is allocated."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"mode": "standalone-store",
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": 0,
|
||||
"local_buffer_size": "1GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
"enable_offload": True,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(extra_config={"preferred_segment": "10.0.0.7:50053"})
|
||||
)
|
||||
|
||||
# setup() receives global_segment_size=0 and the configured local buffer.
|
||||
assert store.setup.call_args.args == (
|
||||
"10.0.0.7",
|
||||
"http://metadata/endpoint",
|
||||
0,
|
||||
1024 * 1024 * 1024,
|
||||
"rdma",
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
# ReplicateConfig is built and carries the preferred_segment.
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
assert w.store_replicate_config.preferred_segment == "10.0.0.7:50053"
|
||||
# Disk-offload staging budget is allocated (enable_offload=True).
|
||||
assert w.disk_offload_buffer_budget_bytes is not None
|
||||
assert w.disk_offload_buffer_budget_bytes > 0
|
||||
|
||||
|
||||
def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
"""embedded + CPU-only: no mode key (defaults to embedded),
|
||||
global_segment_size>0, enable_offload absent, no preferred_segment.
|
||||
This is the PR-40900 baseline recipe."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "4GB",
|
||||
"protocol": "rdma",
|
||||
"device_name": "mlx5_0",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(_make_vllm_config())
|
||||
|
||||
# setup() receives global_segment_size=4 GiB (rank contributes a segment).
|
||||
assert store.setup.call_args.args == (
|
||||
"10.0.0.7",
|
||||
"http://metadata/endpoint",
|
||||
4 * 1024 * 1024 * 1024,
|
||||
4 * 1024 * 1024 * 1024,
|
||||
"rdma",
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
# No preferred_segment — ReplicateConfig is default-constructed (so the
|
||||
# preferred_segment field keeps its default value).
|
||||
assert w.preferred_segment is None
|
||||
assert isinstance(w.store_replicate_config, fake_replicate_config_cls)
|
||||
assert w.store_replicate_config.preferred_segment == ""
|
||||
# No disk budget — enable_offload was absent (defaults to False).
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
Reference in New Issue
Block a user