[Feature][Multimodal] Add cross-encoder caching to Mooncake P2P (#56242)

Signed-off-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zhou ziheng
2026-09-17 13:16:41 +08:00
committed by GitHub
co-authored by Tianyu Guo jiangkuaixue123 Codex
parent 2d8a5c5741
commit 2743dc4f92
15 changed files with 1863 additions and 154 deletions
@@ -0,0 +1,726 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ctypes
import gc
import struct
import threading
import weakref
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config import MultiModalConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import MooncakeECConfig
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakePushSpec,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_ec_connector import (
ECMooncakeConnector,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding import (
backend as store_backend,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding import (
store_client,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
build_embedding_namespace,
make_embedding_key,
)
pytestmark = pytest.mark.cpu_test
KEY = (
"test@embedding@model:model@revision:revision"
"@encoder:encoder@dtype:torch.float32@protocol:v3"
)
SPEC = TensorSpec((2, 2), "torch.float32", 16)
def test_embedding_key_combines_namespace_and_identifier():
config = create_ec_vllm_config(ec_role="ec_producer")
config.ec_transfer_config.ec_connector_extra_config.update(
embedding_cache_prefix="team@cache", embedding_model_identity="shared/model"
)
config.model_config.revision = "rev:1"
config.model_config.dtype = torch.float32
config.model_config.multimodal_config = None
assert make_embedding_key(build_embedding_namespace(config), "image@1") == (
"team@cache@embedding@model:shared/model@revision:rev:1"
"@encoder:encoder:default@dtype:torch.float32@protocol:v3@id:image@1"
)
def test_shared_reuse_requires_content_stable_identifiers():
"""Encoder-only cache-off rendering uses process-local counter identifiers."""
config = create_ec_vllm_config(ec_role="ec_producer")
config.ec_transfer_config.ec_connector = "ECMooncakeConnector"
config.ec_transfer_config.ec_connector_extra_config["cross_encoder_cache"] = True
config.model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=0)
config.use_v2_model_runner = True
config.lora_config = None
with pytest.raises(ValueError, match="mm_processor_cache_gb"):
MooncakeECConfig.from_vllm_config(config)
class MemoryStore:
"""Only the native boundary is faked; codec, resolver and publisher are real."""
def __init__(self):
self.objects: dict[str, bytes] = {}
self.registered: dict[int, int] = {}
def close(self):
return 0
def batch_is_exist(self, keys):
return [int(k in self.objects) for k in keys]
def register_buffer(self, addr, size):
self.registered[addr] = size
return 0
def unregister_buffer(self, addr):
del self.registered[addr]
return 0
def batch_put_from_multi_buffers(self, keys, pointers, sizes, config):
[key], [ptrs], [lengths] = keys, pointers, sizes
self.objects[key] = b"".join(
ctypes.string_at(p, n) for p, n in zip(ptrs, lengths, strict=True)
)
return [0]
def batch_get_into(self, keys, pointers, sizes):
results = []
for key, ptr, size in zip(keys, pointers, sizes, strict=True):
assert any(
base <= ptr and ptr + size <= base + capacity
for base, capacity in self.registered.items()
)
obj = self.objects.get(key)
if obj is None:
results.append(-704)
elif len(obj) > size:
results.append(-600)
else:
ctypes.memmove(ptr, obj, len(obj))
results.append(len(obj))
return results
@pytest.fixture
def backend():
instance = store_backend.MooncakeEmbeddingStoreBackend(
store_client.MooncakeEmbeddingStoreClient(
MemoryStore(), read_buffer_bytes=1024
),
KEY,
max_pending_items=32,
max_pending_bytes=2 * 1024**3,
)
yield instance
# Some tests deliberately poison I/O; always stop the Python executor.
instance._executor.shutdown(wait=True)
if not instance.store_client._poisoned and (
instance.store_client._read_buffer is not None
or instance.store_client._put_header is not None
):
instance.store_client.close()
def test_tensor_round_trip(backend):
client, native = backend.store_client, backend.store_client.store
native.register_buffer = MagicMock(wraps=native.register_buffer)
header_ptr = None
for index, dtype in enumerate((torch.float16, torch.bfloat16, torch.float32)):
tensor = torch.arange(6, dtype=dtype).reshape(index + 1, -1)
key = make_embedding_key(KEY, str(dtype))
client.put_tensor(key, tensor)
pointer = ctypes.addressof(client._put_header)
header_ptr = header_ptr or pointer
assert pointer == header_ptr
expected = TensorSpec(tuple(tensor.shape), str(dtype), tensor.nbytes)
with torch.device("meta"):
loaded = client.load_tensors({key: expected}, "cpu")[key]
assert client._read_buffer.device.type == "cpu"
assert loaded.dtype == dtype and torch.equal(loaded, tensor)
assert len(native.objects[key]) == 24 + tensor.nbytes
assert (
sum(c.args[0] == header_ptr for c in native.register_buffer.call_args_list) == 1
)
assert len(native.registered) == 2
@pytest.mark.parametrize("capacity", [80, 120])
def test_batch_reads_reuse_registration_without_aliasing_outputs(backend, capacity):
client, native = backend.store_client, backend.store_client.store
client._read_buffer_bytes = capacity
tensors = {key: torch.full((2, 2), float(i)) for i, key in enumerate("abc")}
for key, tensor in tensors.items():
client.put_tensor(key, tensor)
native.register_buffer = MagicMock(wraps=native.register_buffer)
native.unregister_buffer = MagicMock(wraps=native.unregister_buffer)
native.batch_get_into = MagicMock(wraps=native.batch_get_into)
expected = dict.fromkeys(tensors, SPEC)
first = client.load_tensors(expected, "cpu")
for key, tensor in tensors.items():
native.objects[key] = native.objects[key][:24] + struct.pack(
"<4f", *[tensor[0, 0].item() + 10] * 4
)
second = client.load_tensors(expected, "cpu")
assert all(torch.equal(first[key], tensor) for key, tensor in tensors.items())
assert all(torch.equal(second[key], tensor + 10) for key, tensor in tensors.items())
assert native.batch_get_into.call_count == (4 if capacity == 80 else 2)
native.register_buffer.assert_called_once()
native.unregister_buffer.assert_not_called()
client.close()
assert native.unregister_buffer.call_count == 2
assert not native.registered
@pytest.mark.parametrize("mode", ["miss", "oversized"])
def test_unloadable_inputs_do_not_allocate_staging(backend, mode):
client, native = backend.store_client, backend.store_client.store
client._read_buffer_bytes = 39
if mode == "oversized":
client.put_tensor(make_embedding_key(KEY, "a"), torch.ones(2, 2))
native.register_buffer = MagicMock(wraps=native.register_buffer)
assert not backend.resolve_inputs({"a": SPEC}, {}, "cpu")
assert client._read_buffer is None
if mode == "miss":
assert client._put_header is None
client.close()
native.register_buffer.assert_not_called()
@pytest.mark.parametrize(
"results", [[40, -800], [40], [40, True], [[40], [40]], None, [40, 41]]
)
def test_unsafe_batch_result_retains_whole_staging(backend, results):
client, native = backend.store_client, backend.store_client.store
native.batch_get_into = MagicMock(return_value=results)
native.unregister_buffer = MagicMock(wraps=native.unregister_buffer)
for _ in range(2):
with pytest.raises(store_client.EmbeddingStoreError, match="unconfirmed"):
client.load_tensors(dict.fromkeys("ab", SPEC), "cpu")
native.batch_get_into.assert_called_once()
with pytest.raises(store_client.EmbeddingStoreError, match="unconfirmed"):
client.close()
native.unregister_buffer.assert_not_called()
assert client._read_buffer is client._unsafe_owners[0]
@pytest.mark.parametrize("failure", ["allocation", "copy", "record", "completion"])
def test_copy_failure_preserves_staging_lifetime(backend, failure):
"""Emulate CUDA allocation/event boundaries without requiring a GPU."""
client = backend.store_client
for key in "ab":
client.put_tensor(key, torch.ones(2, 2))
allocate, copy = torch.empty, torch.Tensor.copy_
targets: list[torch.Tensor] = []
copies: list[torch.Tensor] = []
ready = MagicMock()
if failure == "record":
ready.record.side_effect = RuntimeError("record failure")
elif failure == "completion":
ready.synchronize.side_effect = RuntimeError("copy failure")
def allocate_on_cpu(*args, **kwargs):
is_target = kwargs.get("device") == torch.device("cuda")
if is_target and targets and failure == "allocation":
raise torch.OutOfMemoryError("allocation failure")
kwargs.update(device="cpu", pin_memory=False)
tensor = allocate(*args, **kwargs)
if is_target:
targets.append(tensor)
return tensor
def record_copy(target, source, **kwargs):
if copies and failure == "copy":
raise RuntimeError("copy failure")
copies.append(target)
return copy(target, source, **kwargs)
with (
patch.object(torch, "empty", side_effect=allocate_on_cpu),
patch.object(torch, "Event", return_value=ready),
patch.object(torch.accelerator, "current_stream"),
patch.object(torch.Tensor, "copy_", new=record_copy),
pytest.raises(RuntimeError, match="allocation failure|unconfirmed"),
):
client.load_tensors(dict.fromkeys("ab", SPEC), "cuda")
if failure == "allocation":
assert not copies and not client._poisoned
else:
assert len(copies) == (1 if failure == "copy" else 2)
assert client._poisoned
assert any(owner is client._read_buffer for owner in client._unsafe_owners)
assert any(owner is ready for owner in client._unsafe_owners)
assert all(
any(owner is target for owner in client._unsafe_owners)
for target in targets
)
with pytest.raises(store_client.EmbeddingStoreError, match="unconfirmed"):
client.close()
def test_non_matrix_output_is_rejected_before_registration(backend):
client = backend.store_client
client.store.register_buffer = MagicMock()
with pytest.raises(store_client.EmbeddingStoreOperationError, match="2D"):
client.put_tensor(make_embedding_key(KEY, "a"), torch.ones(2, 2, 2))
client.store.register_buffer.assert_not_called()
def make_worker(backend):
from vllm.distributed.ec_transfer.ec_connector.mooncake.worker import (
ECMooncakeWorker,
)
worker = object.__new__(ECMooncakeWorker)
worker.is_producer = True
worker.is_consumer = False
worker._output_store = backend
worker._buffer_device = "cpu"
worker._producer_pushes = MagicMock()
worker._producer_pushes.reserve.return_value = (MagicMock(), True)
worker._producer_pushes.poll.return_value = []
worker._producer_pushes.cancel_requests.return_value = []
worker._producer_pushes.pending = False
worker._control_executor = MagicMock()
worker._consumer_memory = MagicMock()
worker._consumer_memory.drain_reclaimed.return_value = set()
worker._completed_loads = set()
worker._failed_loads = set()
worker._failed_saves = set()
worker._push_ready = threading.Event()
worker._flush_pending_pushes = MagicMock()
worker._bind_push_source = MagicMock()
connector = object.__new__(ECMooncakeConnector)
connector._worker = worker
connector._role = ECConnectorRole.WORKER
connector._is_producer = True
connector._is_consumer = False
connector._connector_metadata = None
return connector
@pytest.mark.parametrize(
"mode",
[
"miss",
"legacy",
"hit",
"partial",
"get-failure",
"short-data",
"lease-expired",
"registration-failure",
],
)
def test_reuse_pipeline(backend, mode):
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.v1.worker.gpu.ec_connector import ActiveECConnector
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.model_states.interface import ModelState
native = backend.store_client.store
values = {"a": 1.0, "b": 2.0, "c": 3.0}
if mode == "legacy":
old_namespace = KEY.replace("@protocol:v3", "@protocol:v2")
for key in "abc":
native.objects[make_embedding_key(old_namespace, key)] = b"old format"
if mode == "partial":
hits = {"a", "c"}
elif mode not in ("miss", "legacy"):
hits = set("abc")
else:
hits = set()
for key in hits:
backend.store_client.put_tensor(
make_embedding_key(KEY, key), torch.full((2, 2), values[key])
)
if mode == "short-data":
key = make_embedding_key(KEY, "b")
native.objects[key] = native.objects[key][:-1]
hits.remove("b")
if mode in ("get-failure", "lease-expired"):
read = native.batch_get_into
def failed_read(keys, pointers, sizes):
results = read(keys, pointers, sizes)
results[keys.index(make_embedding_key(KEY, "b"))] = (
-707 if mode == "lease-expired" else -704
)
return results
native.batch_get_into = failed_read
hits.remove("b")
if mode == "registration-failure":
native.register_buffer = MagicMock(return_value=-500)
hits.clear()
connector = make_worker(backend)
metadata = ECMooncakeConnectorMetadata(
store_candidates=dict.fromkeys("abc", SPEC),
pushes=[
ECMooncakePushSpec(
mm_hash=k,
shape=(2, 2),
dtype="float32",
nbytes=16,
consumer_zmq="ipc:///tmp/test",
transfer_id=k,
request_id="req",
)
for k in "abc"
],
)
cache = EncoderCache()
cache.mm_features["req"] = [
MultiModalFeatureSpec(
data=MultiModalKwargsItem({}),
modality="image",
identifier=k,
mm_position=PlaceholderRange(offset=i * 2, length=2),
)
for i, k in enumerate("abc")
]
runner = EncoderRunner(
model=None,
max_num_tokens=8,
hidden_size=2,
encoder_cache=cache,
dtype=torch.float32,
device=torch.device("cpu"),
)
runner.enable_timing = False
missing = [k for k in "abc" if k not in hits]
runner.execute_mm_encoder = MagicMock(
return_value=[torch.full((2, 2), values[k] + 10) for k in missing]
)
state = SimpleNamespace(encoder_runner=runner, encoder_cache=cache)
with patch(
"vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector
):
adapter = ActiveECConnector(SimpleNamespace(), cache.encoder_outputs)
with adapter.maybe_get_output(
SimpleNamespace(ec_connector_metadata=metadata, finished_req_ids=set())
) as output:
assert runner.prepare_mm_inputs({"req": [0, 1, 2]})[0] == missing
ModelState.execute_mm_encoder(state, {"req": [0, 1, 2]})
assert runner.execute_mm_encoder.call_count == bool(missing)
assert output.ec_connector_worker_meta.pending_saves is False
assert all(
torch.equal(
cache.encoder_outputs[k],
torch.full((2, 2), values[k] + (0 if k in hits else 10)),
)
for k in "abc"
)
assert {
c.args[1] for c in connector._worker._bind_push_source.call_args_list
} == set("abc")
backend.shutdown()
assert not native.registered
if mode in ("miss", "legacy"):
other = store_backend.MooncakeEmbeddingStoreBackend(
store_client.MooncakeEmbeddingStoreClient(native, read_buffer_bytes=1024),
KEY,
max_pending_items=32,
max_pending_bytes=2 * 1024**3,
)
try:
loaded: dict[str, torch.Tensor] = {}
assert other.resolve_inputs(
dict.fromkeys("abc", SPEC),
loaded,
"cpu",
) == set("abc")
assert all(torch.equal(loaded[k], cache.encoder_outputs[k]) for k in "abc")
finally:
other.shutdown()
@pytest.mark.parametrize(
"mismatch",
["revision", "shape", "dtype", "magic", "short-header", "short-data", "long-data"],
)
def test_incompatible_output_is_a_miss(backend, mismatch):
key = make_embedding_key(KEY, "a")
dtype = torch.float16 if mismatch == "dtype" else torch.float32
backend.store_client.put_tensor(key, torch.ones(2, 2, dtype=dtype))
if mismatch == "revision":
backend.namespace = KEY.replace("@revision:revision", "@revision:other")
objects = backend.store_client.store.objects
if mismatch == "magic":
objects[key] = b"\0" * 4 + objects[key][4:]
elif mismatch == "short-header":
objects[key] = objects[key][:23]
elif mismatch == "short-data":
objects[key] = objects[key][:-1]
elif mismatch == "long-data":
objects[key] += b"extra"
expected = replace(SPEC, shape=(4, 1)) if mismatch == "shape" else SPEC
outputs: dict[str, torch.Tensor] = {}
assert not backend.resolve_inputs({"a": expected}, outputs, "cpu")
assert not outputs
tensor = torch.ones(2, 2)
backend.store_client.put_tensor(
make_embedding_key(backend.namespace, "valid"), tensor
)
assert backend.resolve_inputs({"valid": SPEC}, outputs, "cpu") == {"valid"}
assert torch.equal(outputs["valid"], tensor)
assert len(backend.store_client.store.registered) == 2
def test_partial_registration_rejection_releases_publication(backend):
"""A rejected header registration must not prevent a later publication."""
native = backend.store_client.store
register = native.register_buffer
put = native.batch_put_from_multi_buffers
native.register_buffer = lambda addr, size: (
-500 if size == 24 else register(addr, size)
)
native.batch_put_from_multi_buffers = MagicMock()
tensor = torch.ones(2, 2)
owner = weakref.ref(tensor)
assert backend._enqueue_save("a", tensor)
storage_owner = weakref.ref(tensor.untyped_storage())
future = backend._pending["a"][0]
with pytest.raises(store_client.EmbeddingStoreOperationError, match="register"):
future.result(timeout=5)
del tensor
gc.collect()
assert owner() is None and not native.registered
native.batch_put_from_multi_buffers.assert_not_called()
backend.resolve_inputs({}, {}, "cpu")
assert backend._pending_bytes == 0
gc.collect()
assert storage_owner() is None # The failed Future is still alive.
assert backend.store_client._put_header is None
native.register_buffer = register
native.batch_put_from_multi_buffers = put
backend.store_client.put_tensor("retry", torch.ones(2, 2))
assert len(native.registered) == 1
@pytest.mark.parametrize(
"operation",
["read-unregister", "header-unregister", "put", "rejected-put", "unregister"],
)
def test_native_io_owners(backend, operation):
"""Uncertain I/O retains owners; completed rejection frees them even via Future."""
native, client = backend.store_client.store, backend.store_client
put = native.batch_put_from_multi_buffers
backend.max_pending_bytes = SPEC.nbytes
tensor = torch.ones(2, 2)
owner = weakref.ref(tensor)
rejected = operation == "rejected-put"
if operation in ("read-unregister", "header-unregister"):
key = make_embedding_key(KEY, "a")
client.put_tensor(key, tensor)
if operation == "read-unregister":
client.load_tensors({key: SPEC}, "cpu")
native.unregister_buffer = MagicMock(return_value=-500)
with pytest.raises(store_client.EmbeddingStoreError, match="unregister"):
client.close()
owner = weakref.ref(
client._read_buffer
if operation == "read-unregister"
else client._put_header
)
else:
if operation == "unregister":
native.unregister_buffer = MagicMock(return_value=-500)
else:
native.batch_put_from_multi_buffers = MagicMock(
return_value=[-1700 if rejected else -800]
)
assert backend._enqueue_save("a", tensor)
future = backend._pending["a"][0]
with pytest.raises(store_client.EmbeddingStoreError):
future.result(timeout=5)
del tensor
gc.collect()
if rejected:
assert owner() is None # A live Future must not retain the tensor.
tensor = torch.ones(4, 4) # Too large for admission after reaping.
caller_owner = weakref.ref(tensor)
backend._step_candidates = {"next"}
backend.save_output("next", tensor)
del tensor
gc.collect()
assert caller_owner() is None
assert backend._pending_bytes == 0 and len(native.registered) == 1
header = client._put_header
native.batch_put_from_multi_buffers = put
client.put_tensor("retry", torch.ones(1, 4))
assert client._put_header is header
else:
assert owner() is not None and native.registered
header = bytes(client._put_header)
with pytest.raises(store_client.EmbeddingStoreError, match="unconfirmed"):
client.put_tensor("retry", torch.ones(1, 4))
assert bytes(client._put_header) == header
with pytest.raises(store_client.EmbeddingStoreError, match="unconfirmed"):
client.close()
def test_publication_budget_charges_and_releases_backing_storage(backend):
tensor = torch.arange(16, dtype=torch.float32)[:4].view(2, 2)
owner = weakref.ref(tensor)
future: Future[None] = Future()
backend.max_pending_items = 1
backend.max_pending_bytes = tensor.nbytes
with patch.object(backend._executor, "submit", return_value=future):
assert not backend._enqueue_save("a", tensor)
backend.max_pending_bytes = 64
assert backend._enqueue_save("a", tensor)
storage_owner = weakref.ref(tensor.untyped_storage())
assert not backend._enqueue_save("b", tensor)
del tensor
gc.collect()
backend.reap() # The final step-end poll precedes publication completion.
assert owner() is not None
backend._save(make_embedding_key(KEY, "a"), backend._pending["a"][1])
future.set_result(None)
assert owner() is None and storage_owner() is not None
def load_after_reaping(*args):
assert not backend._pending and backend._pending_bytes == 0
assert storage_owner() is None
return {}
with patch.object(backend.store_client, "load_tensors", load_after_reaping):
assert not backend.resolve_inputs({"b": SPEC}, {}, "cpu")
@pytest.mark.parametrize("rejected", [False, True])
def test_shared_storage_budget_is_released_after_last_publication(backend, rejected):
batch = torch.arange(16, dtype=torch.float32).view(4, 4)
first: Future[None] = Future()
last: Future[None] = Future()
separate: Future[None] = Future()
backend.max_pending_items = 2
backend.max_pending_bytes = batch.nbytes
with patch.object(backend._executor, "submit", side_effect=[first, last, separate]):
assert backend._enqueue_save("a", batch[:2])
assert backend._enqueue_save("b", batch[2:])
assert backend._pending_bytes == batch.nbytes
assert not backend._enqueue_save("a", batch[:2])
assert not backend._enqueue_save("extra", batch[:1])
if rejected:
first.set_exception(store_client.EmbeddingStoreOperationError("rejected"))
else:
first.set_result(None)
backend.reap()
assert backend._pending_bytes == batch.nbytes
assert not backend._enqueue_save("c", torch.ones(2, 2))
last.set_result(None)
backend.reap()
assert backend._pending_bytes == 0
assert backend._enqueue_save("c", torch.ones(2, 2))
assert backend._pending_bytes == 16
separate.set_result(None)
backend.reap()
assert backend._pending_bytes == 0
@pytest.mark.parametrize("shared_storage", [False, True])
def test_completion_race_preserves_failure_and_reclaims_other_items(
backend, shared_storage
):
class RacingFuture(Future[None]):
def done(self):
observed = super().done()
if not observed:
self.set_exception(store_client.EmbeddingStoreError("uncertain"))
return observed
racing = RacingFuture()
other: Future[None] = Future()
tensor = torch.ones(2, 2)
other_tensor = tensor[:1] if shared_storage else torch.ones(2, 2)
with patch.object(backend._executor, "submit", side_effect=[racing, other]):
assert backend._enqueue_save("race", tensor)
assert backend._enqueue_save("other", other_tensor)
backend.reap()
assert backend._pending_bytes == (16 if shared_storage else 32)
other.set_result(None)
with pytest.raises(store_client.EmbeddingStoreError):
backend.reap()
assert backend._pending_bytes == 16
with pytest.raises(store_client.EmbeddingStoreError):
backend._enqueue_save("later", torch.ones(2, 2))
def test_readiness_and_shutdown_do_not_block_p2p_completion(backend):
ready, waiting, entered = [threading.Event() for _ in range(3)]
original = backend._executor.shutdown
def closing(*args, **kwargs):
entered.set()
return original(*args, **kwargs)
native = backend.store_client.store
def close_store():
assert ready.is_set() and not native.registered
assert backend._pending_bytes == 0
return 0
native.close = MagicMock(side_effect=close_store)
put = native.batch_put_from_multi_buffers
def waiting_put(*args):
waiting.set()
assert ready.wait(5)
assert backend.store_client._put_header is not None and native.registered
return put(*args)
native.batch_put_from_multi_buffers = waiting_put
connector = make_worker(backend)
connector.bind_connector_metadata(
ECMooncakeConnectorMetadata(store_candidates={"a": SPEC})
)
cache: dict[str, torch.Tensor] = {}
with (
patch.object(
store_backend, "_record_tensor_ready_event", return_value=MagicMock()
),
patch.object(backend._executor, "shutdown", side_effect=closing),
ThreadPoolExecutor(max_workers=1) as closer,
):
connector.start_save_caches(encoder_cache=cache)
cache["a"] = torch.arange(4, dtype=torch.float32).view(2, 2)
connector.save_caches(cache, "a")
assert connector.build_connector_worker_meta().pending_saves is False
assert waiting.wait(5) and not backend.store_client.store.objects
close = closer.submit(backend.shutdown)
try:
assert entered.wait(5) and not close.done()
assert not backend._enqueue_save("late", torch.ones(2, 2))
finally:
ready.set()
close.result(timeout=5)
native.close.assert_called_once_with()
assert not backend.store_client.store.registered and backend._pending_bytes == 0
assert backend.store_client.store.objects[make_embedding_key(KEY, "a")][
24:
] == struct.pack("<4f", 0, 1, 2, 3)
@@ -17,6 +17,7 @@ import pytest
import torch
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.distributed import mooncake_store
from vllm.distributed.kv_events import KVEventAggregator
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import (
rdma_utils,
@@ -402,6 +403,7 @@ def _install_fake_mooncake(monkeypatch, store_instance: MagicMock):
fake_store_module = types.ModuleType("mooncake.store")
fake_store_module.MooncakeDistributedStore = lambda: store_instance # type: ignore[attr-defined]
fake_store_module.ReplicateConfig = FakeReplicateConfig # type: ignore[attr-defined]
fake_store_module.ObjectDataType = SimpleNamespace(TENSOR=1) # type: ignore[attr-defined]
fake_mooncake_module = types.ModuleType("mooncake")
fake_mooncake_module.store = fake_store_module # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "mooncake", fake_mooncake_module)
@@ -590,7 +592,7 @@ def test_pool_key_cache_prefix_namespaces_and_disambiguates():
def test_default_local_buffer_size_matches_pr40900():
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
assert worker.DEFAULT_LOCAL_BUFFER_SIZE == 4 * 1024**3
assert mooncake_store.DEFAULT_LOCAL_BUFFER_SIZE == 4 * 1024**3
def test_get_requester_local_hostname_prefers_override(monkeypatch):
@@ -2957,6 +2959,7 @@ def test_requester_worker_group_semantics_string_true_enables(
fake_store_module = types.ModuleType("mooncake.store")
fake_store_module.MooncakeDistributedStore = lambda: store # type: ignore[attr-defined]
fake_store_module.ReplicateConfig = FakeReplicateConfig # type: ignore[attr-defined]
fake_store_module.ObjectDataType = SimpleNamespace(TENSOR=1) # type: ignore[attr-defined]
fake_mooncake_module = types.ModuleType("mooncake")
fake_mooncake_module.store = fake_store_module # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "mooncake", fake_mooncake_module)
@@ -4377,10 +4380,10 @@ def test_config_defaults_to_embedded():
"""A JSON without explicit mode parses as embedded with 4 GiB segment."""
cfg = _make_config()
assert cfg.mode == "embedded"
assert cfg.global_segment_size == worker.DEFAULT_GLOBAL_SEGMENT_SIZE
assert cfg.local_buffer_size == worker.DEFAULT_LOCAL_BUFFER_SIZE
assert cfg.global_segment_size == mooncake_store.DEFAULT_GLOBAL_SEGMENT_SIZE
assert cfg.local_buffer_size == mooncake_store.DEFAULT_LOCAL_BUFFER_SIZE
assert cfg.enable_offload is False
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
assert cfg.tenant_id == mooncake_store.DEFAULT_TENANT_ID
def test_config_pr40900_unchanged(tmp_path):
@@ -4402,7 +4405,7 @@ def test_config_pr40900_unchanged(tmp_path):
assert cfg.global_segment_size == 4 * 1024**3
assert cfg.local_buffer_size == 4 * 1024**3
assert cfg.enable_offload is False
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
assert cfg.tenant_id == mooncake_store.DEFAULT_TENANT_ID
def test_config_from_file_normalizes_tenant_id(tmp_path):
@@ -4432,7 +4435,7 @@ def test_config_from_file_normalizes_empty_tenant_id_to_default(tmp_path):
cfg = worker.MooncakeStoreConfig.from_file(config_path)
assert cfg.tenant_id == worker.DEFAULT_TENANT_ID
assert cfg.tenant_id == mooncake_store.DEFAULT_TENANT_ID
def test_config_from_file_rejects_non_string_tenant_id(tmp_path):
@@ -4584,7 +4587,20 @@ def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
assert w.disk_offload_buffer_budget_bytes is None
def test_topology_forwards_non_default_tenant_id(tmp_path, monkeypatch):
def _create_store_client_for_tenant_test(consumer):
if consumer == "ec":
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.store_client import ( # noqa: E501
create_mooncake_embedding_store_client,
)
return create_mooncake_embedding_store_client()
return worker.MooncakeStoreWorker(
_make_vllm_config(rank=1), _make_kv_cache_config()
)
@pytest.mark.parametrize("consumer", ["kv", "ec"])
def test_topology_forwards_non_default_tenant_id(tmp_path, monkeypatch, consumer):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
@@ -4605,12 +4621,13 @@ def test_topology_forwards_non_default_tenant_id(tmp_path, monkeypatch):
),
)
worker.MooncakeStoreWorker(_make_vllm_config(rank=1), _make_kv_cache_config())
_create_store_client_for_tenant_test(consumer)
assert store.setup.call_args.kwargs == {"tenant_id": "tenant-a"}
def test_non_default_tenant_preserves_setup_type_error(tmp_path, monkeypatch):
@pytest.mark.parametrize("consumer", ["kv", "ec"])
def test_non_default_tenant_preserves_setup_type_error(tmp_path, monkeypatch, consumer):
store = MagicMock()
setup_error = TypeError(
"setup(): incompatible function arguments; "
@@ -4636,7 +4653,7 @@ def test_non_default_tenant_preserves_setup_type_error(tmp_path, monkeypatch):
)
with pytest.raises(TypeError) as exc_info:
worker.MooncakeStoreWorker(_make_vllm_config(rank=1), _make_kv_cache_config())
_create_store_client_for_tenant_test(consumer)
assert exc_info.value is setup_error