[KV Connector] Add decode offloading to Mooncake Store consumers (#52466)
Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn> Signed-off-by: Guanyi Chen <939416532@qq.com> Co-authored-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
This commit is contained in:
@@ -136,6 +136,25 @@ vllm serve meta-llama/Llama-3.1-8B-Instruct \
|
||||
}'
|
||||
```
|
||||
|
||||
To also offload newly completed decode KV blocks, add the following extra
|
||||
configuration to the decoder's `MooncakeStoreConnector` entry. This currently
|
||||
requires the prefill and decode instances to use the same tensor-parallel size
|
||||
and compatible KV-cache topology.
|
||||
|
||||
When decode processing starts, the consumer checks the block-aligned prompt
|
||||
prefix and fills any blocks missing from the Store. Subsequent saves append
|
||||
newly completed decode blocks. This keeps a complete, reusable prefix in the
|
||||
Store and also covers deployments where prompt KV is delivered directly by
|
||||
`MooncakeConnector` instead of through the Store.
|
||||
|
||||
```json
|
||||
{
|
||||
"kv_connector_extra_config": {
|
||||
"save_decode_cache": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Proxy:**
|
||||
|
||||
A disaggregation proxy is required to route requests between prefiller and decoder nodes. The proxy assigns `do_remote_prefill=True` / `do_remote_decode=True` to coordinate P2P transfer via `MooncakeConnector`. Refer to the [MooncakeConnector usage guide](mooncake_connector_usage.md) for proxy setup details.
|
||||
@@ -230,6 +249,10 @@ Strict isolation requires a Mooncake master started with `--enable_multi_tenants
|
||||
- `enable_cross_layers_blocks` (bool): Enable cross-layer block packing for reduced store operations. Default: `false`.
|
||||
- `lookup_rpc_port` (int): Custom port for the ZMQ lookup RPC socket. Default: `0`.
|
||||
- `cache_prefix` (str): Namespace prepended to every store key. Lets separate deployments share one Mooncake master without polluting each other — instances configured with different prefixes never see each other's cached blocks, even for identical prompts. All instances that should share a prefix cache must use the same value. Default: `""` (no prefix; keys are byte-identical to the unprefixed format).
|
||||
- `save_decode_cache` (bool): Enable offloading decode tokens' KV cache. A `kv_consumer` does not save during prefill; when decode starts, it fills any missing block-aligned prompt prefix before appending completed decode blocks. Default: `false`.
|
||||
|
||||
Decode offloading uses the existing TP-rank key namespace. Cross-TP sharing is
|
||||
not yet supported.
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
@@ -16,10 +18,16 @@ from vllm.v1.core.block_pool import BlockPool
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
*, hash_block_size: int = 16, enable_partial_hash_hits: bool = False
|
||||
*,
|
||||
hash_block_size: int = 16,
|
||||
enable_partial_hash_hits: bool = False,
|
||||
kv_role: str = "kv_both",
|
||||
save_decode_cache: bool = False,
|
||||
) -> MooncakeStoreScheduler:
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.kv_role = kv_role
|
||||
scheduler.save_decode_cache = save_decode_cache
|
||||
scheduler.enable_kv_events = False
|
||||
scheduler.lookup_async = False
|
||||
scheduler.enable_lookup = True
|
||||
scheduler._block_size = 16
|
||||
@@ -56,6 +64,74 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
)
|
||||
|
||||
|
||||
def _make_decode_scheduler_output(
|
||||
*, num_computed_tokens: int, num_scheduled_tokens: int = 1
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
# The block that becomes full was allocated on an earlier step.
|
||||
new_block_ids=[()],
|
||||
num_computed_tokens=[num_computed_tokens],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
|
||||
|
||||
def _make_new_scheduler_output() -> SimpleNamespace:
|
||||
request = SimpleNamespace(
|
||||
req_id="req-0",
|
||||
num_computed_tokens=0,
|
||||
prompt_token_ids=list(range(32)),
|
||||
prefill_token_ids=None,
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=[b"h0", b"h1"],
|
||||
)
|
||||
return SimpleNamespace(
|
||||
request=request,
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[request],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 32},
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_only_tracks_token_ids_for_kv_events():
|
||||
for enable_kv_events in (False, True):
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.enable_kv_events = enable_kv_events
|
||||
scheduler._unfinished_requests["req-0"] = (
|
||||
_make_new_scheduler_output().request,
|
||||
([0, 1],),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(_make_new_scheduler_output())
|
||||
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
req_meta = meta.requests[0]
|
||||
if enable_kv_events:
|
||||
assert tracker.token_ids == list(range(32))
|
||||
assert req_meta.token_ids == list(range(32))
|
||||
assert req_meta.token_ids_start == 0
|
||||
tracker.token_ids.append(99)
|
||||
assert req_meta.token_ids == list(range(32))
|
||||
else:
|
||||
assert tracker.token_ids is None
|
||||
assert req_meta.token_ids is None
|
||||
|
||||
|
||||
def _make_preemption_scheduler_output():
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
@@ -103,6 +179,29 @@ def _add_unfinished_request(
|
||||
)
|
||||
|
||||
|
||||
def _setup_decode_request(
|
||||
*,
|
||||
kv_role: str = "kv_consumer",
|
||||
save_decode_cache: bool = False,
|
||||
token_len: int = 47,
|
||||
) -> tuple[MooncakeStoreScheduler, RequestTracker]:
|
||||
scheduler = _make_bare_scheduler(
|
||||
kv_role=kv_role, save_decode_cache=save_decode_cache
|
||||
)
|
||||
token_ids = list(range(token_len + 1))
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=token_ids,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=32,
|
||||
)
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
tracker.token_len = token_len
|
||||
tracker.allocated_block_ids = ([0, 1, 2],)
|
||||
tracker.token_ids = token_ids[:token_len]
|
||||
return scheduler, tracker
|
||||
|
||||
|
||||
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
|
||||
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
|
||||
# yet, so the tracker's token_len does not advance and num_tokens_to_save
|
||||
@@ -149,6 +248,92 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_role", ["kv_consumer", "kv_both"])
|
||||
def test_decode_tracking_is_skipped_by_default(kv_role):
|
||||
scheduler, tracker = _setup_decode_request(kv_role=kv_role)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_decode_scheduler_output(num_computed_tokens=47)
|
||||
)
|
||||
|
||||
assert meta.requests == []
|
||||
assert tracker.token_len == 47
|
||||
assert tracker.num_saved_tokens == 32
|
||||
assert tracker.allocated_block_ids == ([0, 1, 2],)
|
||||
assert tracker.token_ids == list(range(47))
|
||||
|
||||
|
||||
def test_fresh_consumer_first_decode_save_can_backfill_missing_prompt():
|
||||
scheduler = _make_bare_scheduler(
|
||||
kv_role="kv_consumer",
|
||||
save_decode_cache=True,
|
||||
)
|
||||
scheduler.enable_kv_events = True
|
||||
new_output = _make_new_scheduler_output()
|
||||
request = new_output.request
|
||||
request.block_ids = ([0, 1, 2],)
|
||||
request.block_hashes = [b"h0", b"h1", b"h2"]
|
||||
request.all_token_ids = list(range(48))
|
||||
request.num_output_placeholders = 0
|
||||
scheduler._unfinished_requests["req-0"] = (request, request.block_ids)
|
||||
|
||||
# The consumer does not save during prefill, so its first decode save must
|
||||
# still cover the prompt. The worker deduplicates prompt blocks already in
|
||||
# the Store and fills any that are missing, preserving a complete prefix.
|
||||
assert scheduler.build_connector_meta(new_output).requests == []
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
|
||||
# The first decode step checks/saves the block-aligned prompt. The worker's
|
||||
# Store dedup turns this into a no-op when the producer already saved it.
|
||||
[prompt_meta] = scheduler.build_connector_meta(
|
||||
_make_decode_scheduler_output(num_computed_tokens=32)
|
||||
).requests
|
||||
assert prompt_meta.token_len_chunk == 32
|
||||
assert prompt_meta.token_ids_start == 0
|
||||
assert prompt_meta.token_ids == list(range(32))
|
||||
|
||||
for num_computed_tokens in range(33, 48):
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_decode_scheduler_output(
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
)
|
||||
)
|
||||
if num_computed_tokens < 47:
|
||||
assert meta.requests == []
|
||||
|
||||
[req_meta] = meta.requests
|
||||
assert req_meta.token_len_chunk == 48
|
||||
assert req_meta.token_ids_start == 32
|
||||
assert req_meta.token_ids == list(range(32, 48))
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_len, saved_tokens", [(46, 32), (47, 48)])
|
||||
def test_consumer_saves_only_full_decode_blocks(token_len, saved_tokens):
|
||||
scheduler, tracker = _setup_decode_request(
|
||||
save_decode_cache=True, token_len=token_len
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_decode_scheduler_output(num_computed_tokens=token_len)
|
||||
)
|
||||
|
||||
assert tracker.token_len == token_len + 1
|
||||
assert tracker.num_saved_tokens == saved_tokens
|
||||
if saved_tokens == 32:
|
||||
assert meta.requests == []
|
||||
else:
|
||||
[req_meta] = meta.requests
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 48
|
||||
assert req_meta.block_ids == ([0, 1, 2],)
|
||||
assert req_meta.token_ids == list(range(32, 48))
|
||||
assert req_meta.token_ids_start == 32
|
||||
tracker.token_ids.append(999)
|
||||
assert req_meta.token_ids == list(range(32, 48))
|
||||
|
||||
|
||||
def test_preemption_resets_tracker():
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
|
||||
|
||||
class _RecordingBlockHashes:
|
||||
@@ -230,6 +230,7 @@ def _make_vllm_config(
|
||||
extra_config: dict[str, object] | None = None,
|
||||
rank: int = 0,
|
||||
decode_context_parallel_size: int = 1,
|
||||
kv_role: str = "kv_both",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_config=_FakeModelConfig(),
|
||||
@@ -240,7 +241,9 @@ def _make_vllm_config(
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
prefill_context_parallel_size=1,
|
||||
),
|
||||
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
|
||||
kv_transfer_config=_FakeKVTransferConfig(
|
||||
kv_role=kv_role, extra_config=extra_config
|
||||
),
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
@@ -943,15 +946,20 @@ def test_stale_store_job_cannot_touch_a_reused_request_id():
|
||||
live = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
live.store_job_id = 2
|
||||
thread.add_request(live)
|
||||
thread._retry_token_ids["req-a"] = (32, list(range(32, 64)))
|
||||
|
||||
thread.finish_store_job(stale)
|
||||
thread._record_saved(stale, 64)
|
||||
thread._mark_request_skipped_for_pressure(stale)
|
||||
thread._update_retry_token_ids(stale, False, 0, list(range(32)))
|
||||
thread._update_retry_token_ids(stale, True, 0, None)
|
||||
|
||||
assert thread.is_live_store_job(live)
|
||||
assert not thread.is_live_store_job(stale)
|
||||
assert thread._saved_offset.get("req-a") is None
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
assert thread._get_retry_token_ids(stale) is None
|
||||
assert thread._get_retry_token_ids(live) == (32, list(range(32, 64)))
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
@@ -1655,6 +1663,39 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
assert w.disk_offload_buffer_budget_bytes is None
|
||||
|
||||
|
||||
def test_save_decode_cache_keeps_transfer_path_enabled(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"MOONCAKE_CONFIG_PATH",
|
||||
_write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
"metadata_server": "http://metadata/endpoint",
|
||||
"protocol": "tcp",
|
||||
"device_name": "",
|
||||
"master_server_address": "10.0.0.7:50051",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
w = worker.MooncakeStoreWorker(
|
||||
_make_vllm_config(
|
||||
kv_role="kv_consumer",
|
||||
extra_config={
|
||||
"enable_lookup": False,
|
||||
"save_decode_cache": True,
|
||||
},
|
||||
),
|
||||
_make_kv_cache_config(),
|
||||
)
|
||||
|
||||
assert w.can_put is True
|
||||
assert w._capacity_only is False
|
||||
|
||||
|
||||
def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
@@ -1962,7 +2003,7 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, *_args: [256] * len(keys)
|
||||
lambda keys, addrs, sizes, *_args: ([256] * len(keys))
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
@@ -2176,6 +2217,85 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
assert swa_event.block_hashes == [maybe_convert_block_hash(BlockHash(hs[3]))]
|
||||
|
||||
|
||||
def _make_event_store_req(token_len: int, token_ids_start: int = 0) -> ReqMeta:
|
||||
num_blocks = token_len // 16
|
||||
return ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=token_len,
|
||||
block_ids=(list(range(num_blocks)),),
|
||||
block_hashes=[f"a{i}".encode() for i in range(num_blocks)],
|
||||
can_save=True,
|
||||
token_ids=list(range(token_ids_start, token_len)),
|
||||
token_ids_start=token_ids_start,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("saved_offset", "put_step", "exists", "stored_indices", "parent_indices"),
|
||||
[
|
||||
pytest.param(16, 1, [0, 0, 0], [1, 2, 3], [0, 1, 2], id="suffix"),
|
||||
pytest.param(0, 1, [1, 0, 1, 0], [1, 3], [0, 2], id="dedup-holes"),
|
||||
pytest.param(0, 2, [0, 0], [0, 2], [None, 1], id="tp-stride"),
|
||||
],
|
||||
)
|
||||
def test_store_sending_thread_kv_events_use_request_chain_parents(
|
||||
saved_offset, put_step, exists, stored_indices, parent_indices
|
||||
):
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = exists
|
||||
store.batch_put_from_multi_buffers.return_value = [256] * len(stored_indices)
|
||||
thread = _make_store_sending_thread(store, put_step=put_step)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
thread._saved_offset["r0"] = saved_offset
|
||||
_run_store_req(thread, _make_event_store_req(64, saved_offset))
|
||||
|
||||
events = thread.get_kv_events()
|
||||
assert [event.block_hashes for event in events] == [
|
||||
[maybe_convert_block_hash(BlockHash(f"a{i}".encode()))] for i in stored_indices
|
||||
]
|
||||
assert [event.parent_block_hash for event in events] == [
|
||||
(None if i is None else maybe_convert_block_hash(BlockHash(f"a{i}".encode())))
|
||||
for i in parent_indices
|
||||
]
|
||||
assert thread._retry_token_ids == {}
|
||||
|
||||
|
||||
def test_store_sending_thread_kv_events_retry_without_covered_tokens():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(32, 16))
|
||||
|
||||
retry_event, suffix_event = thread.get_kv_events()
|
||||
assert retry_event.token_ids == []
|
||||
assert suffix_event.token_ids == list(range(16, 32))
|
||||
|
||||
|
||||
def test_store_sending_thread_kv_events_recover_suffix_after_put_failure():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = ([0], [0, 0])
|
||||
store.batch_put_from_multi_buffers.side_effect = ([-1], [256, 256])
|
||||
thread = _make_store_sending_thread(store)
|
||||
thread.enable_kv_event = True
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(16))
|
||||
|
||||
assert thread.get_kv_events() == []
|
||||
assert thread._retry_token_ids["r0"] == (0, list(range(16)))
|
||||
|
||||
_run_store_req(thread, _make_event_store_req(32, 16))
|
||||
|
||||
retry_event, suffix_event = thread.get_kv_events()
|
||||
assert retry_event.token_ids == list(range(16))
|
||||
assert suffix_event.token_ids == list(range(16, 32))
|
||||
assert thread._retry_token_ids == {}
|
||||
assert thread._saved_offset["r0"] == 32
|
||||
|
||||
|
||||
def _auto_set_ready_event(*args, **kwargs):
|
||||
"""Side effect for mocked thread constructors that auto-sets ready_event."""
|
||||
for arg in args:
|
||||
@@ -2214,6 +2334,7 @@ def _make_bare_worker(
|
||||
num_gpu_blocks: int = 10,
|
||||
block_size: int = 16,
|
||||
kv_role: str = "kv_both",
|
||||
save_decode_cache: bool = False,
|
||||
) -> mooncake_store_worker.MooncakeStoreWorker:
|
||||
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
|
||||
|
||||
@@ -2227,14 +2348,17 @@ def _make_bare_worker(
|
||||
worker.store = MagicMock()
|
||||
worker.store.register_buffer.return_value = 0
|
||||
worker.kv_role = kv_role
|
||||
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
|
||||
worker._capacity_only = False
|
||||
worker.block_size = block_size
|
||||
worker.tp_rank = 0
|
||||
worker.enable_kv_events = False
|
||||
worker.load_async = True
|
||||
worker.kv_send_thread = None
|
||||
worker.kv_recv_threads = []
|
||||
worker.num_recv_threads = 1
|
||||
worker.recv_request_queue = queue.Queue()
|
||||
worker.finished_store_req = set()
|
||||
worker.tp_size = 1
|
||||
worker.num_kv_head = 1
|
||||
worker.pp_size = 1
|
||||
@@ -2743,6 +2867,36 @@ def test_lookup_applies_swa_mask_before_accessing_hashes():
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("save_decode_cache", [False, True])
|
||||
def test_consumer_starts_send_thread_only_when_put_is_enabled(save_decode_cache):
|
||||
num_blocks = 10
|
||||
tensor = torch.zeros(num_blocks, 64, dtype=torch.float16)
|
||||
|
||||
worker = _make_bare_worker(
|
||||
kv_role="kv_consumer", save_decode_cache=save_decode_cache
|
||||
)
|
||||
_register_with_mocked_threads(worker, {"layer0": tensor})
|
||||
assert (worker.kv_send_thread is not None) == save_decode_cache
|
||||
|
||||
|
||||
def test_putting_consumer_queues_decode_save():
|
||||
w = _make_bare_worker(kv_role="kv_consumer", save_decode_cache=True)
|
||||
send_thread = MagicMock()
|
||||
w.kv_send_thread = send_thread
|
||||
req = _make_store_req("decode-req", [b"h0", b"h1"])
|
||||
req.store_job_id = 1
|
||||
meta = mooncake_store_worker.MooncakeStoreConnectorMetadata(set(), set())
|
||||
meta.add_request(req)
|
||||
|
||||
with patch.object(torch.cuda, "Event") as event_cls:
|
||||
event = event_cls.return_value
|
||||
w.get_finished(set(), meta)
|
||||
|
||||
event.record.assert_called_once_with()
|
||||
send_thread.add_request.assert_called_once_with(req)
|
||||
assert req.current_event is event
|
||||
|
||||
|
||||
def test_register_kv_caches_blocks_first_single_segment():
|
||||
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
|
||||
num_blocks = 10
|
||||
|
||||
@@ -141,12 +141,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
save_decode_cache = extra_config.get("save_decode_cache", False)
|
||||
# Capacity-only: contributes its segment to the store pool but transfers
|
||||
# no KV, so the KV-cache-shape invariants below cannot be reached.
|
||||
self._capacity_only = self.kv_role == "kv_consumer" and not (
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"enable_lookup", True
|
||||
)
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not extra_config.get("enable_lookup", True)
|
||||
and not save_decode_cache
|
||||
)
|
||||
if not self._capacity_only:
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
|
||||
@@ -362,10 +362,11 @@ class ReqMeta:
|
||||
|
||||
can_save: bool | None = None
|
||||
load_spec: LoadSpec | None = None
|
||||
is_last_chunk: bool | None = None
|
||||
current_event: torch.cuda.Event | None = None
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
# Absolute request offset represented by token_ids[0].
|
||||
token_ids_start: int = 0
|
||||
num_prompt_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
@@ -384,14 +385,14 @@ class ReqMeta:
|
||||
load_spec: LoadSpec | None = None,
|
||||
skip_save: bool | None = False,
|
||||
block_hashes: list[BlockHash] | None = None,
|
||||
is_last_chunk: bool | None = None,
|
||||
) -> "ReqMeta | None":
|
||||
"""Create ReqMeta from a RequestTracker."""
|
||||
if block_hashes is None:
|
||||
block_hashes = []
|
||||
input_token_len = tracker.token_len
|
||||
|
||||
chunk_boundary = cdiv(tracker.num_saved_tokens + 1, block_size) * block_size
|
||||
token_ids_start = tracker.num_saved_tokens
|
||||
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
|
||||
num_tokens_to_save = input_token_len // block_size * block_size
|
||||
|
||||
skip_save = skip_save or num_tokens_to_save < chunk_boundary
|
||||
@@ -408,8 +409,10 @@ class ReqMeta:
|
||||
tracker.num_saved_tokens = num_tokens_to_save
|
||||
|
||||
token_ids = None
|
||||
if tracker.token_ids:
|
||||
token_ids = tracker.token_ids
|
||||
if tracker.token_ids and not skip_save:
|
||||
# Scheduler tracking continues while this job is handled by an
|
||||
# asynchronous worker, so metadata must own a stable snapshot.
|
||||
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
|
||||
|
||||
if load_spec is not None and load_spec.can_load:
|
||||
logger.debug(
|
||||
@@ -433,8 +436,8 @@ class ReqMeta:
|
||||
can_save=not skip_save,
|
||||
load_spec=load_spec,
|
||||
block_hashes=block_hashes,
|
||||
is_last_chunk=is_last_chunk,
|
||||
token_ids=token_ids,
|
||||
token_ids_start=token_ids_start,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -63,6 +63,11 @@ class MooncakeStoreScheduler:
|
||||
self.lookup_async = kvc_extra_config.get("lookup_async", False)
|
||||
# Skips lookup CPU cost on instances that never load KV from the store.
|
||||
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
|
||||
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
|
||||
kv_event_config = vllm_config.kv_events_config
|
||||
self.enable_kv_events = bool(
|
||||
kv_event_config and kv_event_config.enable_kv_cache_events
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
@@ -182,7 +187,8 @@ class MooncakeStoreScheduler:
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build connector metadata for this scheduler step."""
|
||||
force_skip_save = self.kv_role == "kv_consumer"
|
||||
is_consumer = self.kv_role == "kv_consumer"
|
||||
can_process_cached = not is_consumer or self.save_decode_cache
|
||||
|
||||
for finished_req_id in scheduler_output.finished_req_ids:
|
||||
self.client.discard(finished_req_id)
|
||||
@@ -227,37 +233,38 @@ class MooncakeStoreScheduler:
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=unfolded_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=prefill_tokens[:num_tokens_to_compute],
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[request.req_id] = request_tracker
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
len(prefill_tokens) // self._block_size * self._block_size
|
||||
)
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=force_skip_save,
|
||||
# A consumer may write decode KV without becoming a prefill
|
||||
# producer. Loads are still carried by the same metadata.
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
is_last_chunk=(request_tracker.token_len >= last_chunk_tokens_num),
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Handle cached (running, or MRV1 resumed-from-preemption) requests
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
if not force_skip_save:
|
||||
if can_process_cached:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if not new_block_ids:
|
||||
continue
|
||||
|
||||
req_meta = None
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if not new_block_ids:
|
||||
continue
|
||||
if isinstance(new_block_ids, tuple):
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
@@ -277,27 +284,33 @@ class MooncakeStoreScheduler:
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=new_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=prefill_tokens[:num_tokens_to_compute].copy(),
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[req_id] = request_tracker
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
len(prefill_tokens) // self._block_size * self._block_size
|
||||
)
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=force_skip_save,
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
is_last_chunk=(
|
||||
request_tracker.token_len >= last_chunk_tokens_num
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Decode/chunked request
|
||||
request_tracker = self._request_trackers[req_id]
|
||||
num_computed_token = cached_reqs.num_computed_tokens[i]
|
||||
# Use the tracker's snapshot of the prefill range so resumed
|
||||
# requests keep saving past the original prompt boundary.
|
||||
prefill_end = request_tracker.prefill_end_tokens
|
||||
is_decode = num_computed_token >= prefill_end
|
||||
if is_decode and not self.save_decode_cache:
|
||||
continue
|
||||
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if req_tuple:
|
||||
@@ -307,30 +320,26 @@ class MooncakeStoreScheduler:
|
||||
num_current_tokens : num_current_tokens + num_new_tokens
|
||||
]
|
||||
request_tracker.token_len += len(new_token_ids)
|
||||
if request_tracker.token_ids is not None:
|
||||
request_tracker.token_ids.extend(new_token_ids)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Request {req_id} is not in _unfinished_requests"
|
||||
)
|
||||
num_computed_token = cached_reqs.num_computed_tokens[i]
|
||||
# Use the tracker's snapshot of the prefill range so resumed
|
||||
# requests keep saving past the original prompt boundary.
|
||||
prefill_end = request_tracker.prefill_end_tokens
|
||||
if num_computed_token >= prefill_end:
|
||||
# A block is usually allocated before the step that fills
|
||||
# it, so reaching a save boundary does not imply that this
|
||||
# step has new block ids.
|
||||
if new_block_ids:
|
||||
request_tracker.update(new_block_ids)
|
||||
if is_consumer and not is_decode:
|
||||
continue
|
||||
request_tracker.update(new_block_ids)
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
prefill_end // self._block_size * self._block_size
|
||||
)
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=None,
|
||||
skip_save=force_skip_save,
|
||||
skip_save=False,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
is_last_chunk=(
|
||||
request_tracker.token_len >= last_chunk_tokens_num
|
||||
),
|
||||
)
|
||||
|
||||
if req_meta is not None:
|
||||
@@ -370,7 +379,7 @@ class MooncakeStoreScheduler:
|
||||
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
|
||||
# save; can_save=True takes the normal enqueue path).
|
||||
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
|
||||
if step_partial_tails and not force_skip_save:
|
||||
if step_partial_tails and not is_consumer:
|
||||
pending = dict(step_partial_tails)
|
||||
for req_meta in meta.requests:
|
||||
if req_meta.can_save:
|
||||
|
||||
@@ -530,6 +530,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
# Per-request high-water mark of tokens actually persisted; the next
|
||||
# batch resumes here, so pressure-skipped or failed ranges are retried.
|
||||
self._saved_offset: dict[str, int] = {}
|
||||
# Retained only after a failed store so retry events can recover the
|
||||
# token suffix without full snapshots on the normal path.
|
||||
self._retry_token_ids: dict[str, tuple[int, list[int]]] = {}
|
||||
|
||||
def add_request(self, request: ReqMeta) -> None:
|
||||
# Register before enqueueing so a job is never picked up unledgered.
|
||||
@@ -552,6 +555,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
del self.stored_requests[req_id]
|
||||
self._skip_store_requests.discard(req_id)
|
||||
self._saved_offset.pop(req_id, None)
|
||||
self._retry_token_ids.pop(req_id, None)
|
||||
|
||||
def finish_store_job(self, req_meta: ReqMeta) -> None:
|
||||
"""Retire a job from the ledger and report its blocks as no longer read.
|
||||
@@ -585,6 +589,36 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
|
||||
self._saved_offset[req_meta.req_id] = token_len
|
||||
|
||||
def _get_retry_token_ids(self, req_meta: ReqMeta) -> tuple[int, list[int]] | None:
|
||||
"""Return retry state only if this store job is still live."""
|
||||
with self.done_task_lock:
|
||||
if req_meta.store_job_id not in self.stored_requests.get(
|
||||
req_meta.req_id, ()
|
||||
):
|
||||
return None
|
||||
return self._retry_token_ids.get(req_meta.req_id)
|
||||
|
||||
def _update_retry_token_ids(
|
||||
self,
|
||||
req_meta: ReqMeta,
|
||||
save_completed: bool,
|
||||
token_ids_start: int,
|
||||
event_token_ids: list[int] | None,
|
||||
) -> None:
|
||||
"""Update retry state without letting a stale job touch a reused ID."""
|
||||
with self.done_task_lock:
|
||||
if req_meta.store_job_id not in self.stored_requests.get(
|
||||
req_meta.req_id, ()
|
||||
):
|
||||
return
|
||||
if save_completed:
|
||||
self._retry_token_ids.pop(req_meta.req_id, None)
|
||||
elif event_token_ids is not None:
|
||||
self._retry_token_ids[req_meta.req_id] = (
|
||||
token_ids_start,
|
||||
event_token_ids,
|
||||
)
|
||||
|
||||
def _should_skip_request(self, req_id: str) -> bool:
|
||||
with self.done_task_lock:
|
||||
return self._store_pressure_active and req_id in self._skip_store_requests
|
||||
@@ -785,18 +819,30 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
# The single `finally` is the only way out, so the scheduler releases
|
||||
# this job's GPU block references however the job ends.
|
||||
save_completed = False
|
||||
token_len = 0
|
||||
req_id = req_meta.req_id
|
||||
event_token_ids = req_meta.token_ids
|
||||
token_ids_start = req_meta.token_ids_start
|
||||
try:
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
|
||||
# which is also ``store_mask``'s precondition.
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if not self.is_live_store_job(req_meta):
|
||||
return
|
||||
|
||||
if self.enable_kv_event:
|
||||
retry_token_ids = self._get_retry_token_ids(req_meta)
|
||||
if retry_token_ids is not None and event_token_ids is not None:
|
||||
retry_start, retry_ids = retry_token_ids
|
||||
if retry_start + len(retry_ids) == token_ids_start:
|
||||
event_token_ids = retry_ids + event_token_ids
|
||||
token_ids_start = retry_start
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -852,6 +898,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_meta, token_len)
|
||||
save_completed = True
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
@@ -878,6 +925,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
if not missing_indices:
|
||||
self._record_saved(req_meta, token_len)
|
||||
save_completed = True
|
||||
return
|
||||
|
||||
if len(missing_indices) != len(keys):
|
||||
@@ -927,12 +975,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
addrs.extend(group_addrs)
|
||||
sizes.extend(group_sizes)
|
||||
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
if self.enable_kv_event:
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [
|
||||
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
|
||||
]
|
||||
token_ids_end = token_ids_start + len(event_token_ids or ())
|
||||
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
@@ -940,13 +987,24 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
db = self.token_databases[g_idx]
|
||||
if self.enable_kv_event:
|
||||
token_ids = (
|
||||
req_meta.token_ids[s:e]
|
||||
if req_meta.token_ids is not None
|
||||
else None
|
||||
event_token_ids[s - token_ids_start : e - token_ids_start]
|
||||
if event_token_ids is not None
|
||||
and token_ids_start <= s
|
||||
and e <= token_ids_end
|
||||
else []
|
||||
)
|
||||
stored_event = BlockStored(
|
||||
block_hashes=[new_block_hashes[idx]],
|
||||
parent_block_hash=prev_key_per_group.get(g_idx),
|
||||
# Derive the direct predecessor from the unfiltered
|
||||
# request chain. Adjacent PUTs need not be adjacent in
|
||||
# that chain after Store dedup, masks, or TP striding.
|
||||
parent_block_hash=(
|
||||
maybe_convert_block_hash(
|
||||
req_meta.block_hashes[s // db.hash_block_size - 1]
|
||||
)
|
||||
if s > 0
|
||||
else None
|
||||
),
|
||||
token_ids=token_ids,
|
||||
block_size=db.block_size,
|
||||
lora_id=None,
|
||||
@@ -955,7 +1013,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_idx=g_idx,
|
||||
)
|
||||
stored_events.append(stored_event)
|
||||
prev_key_per_group[g_idx] = new_block_hashes[idx]
|
||||
|
||||
if current_event is not None:
|
||||
current_event.synchronize()
|
||||
@@ -984,6 +1041,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
if failed:
|
||||
failed_codes = set(res[i] for i in failed)
|
||||
if self.enable_kv_event:
|
||||
failed_indices = set(failed)
|
||||
stored_events = [
|
||||
event
|
||||
for i, event in enumerate(stored_events)
|
||||
if i not in failed_indices
|
||||
]
|
||||
logger.warning(
|
||||
"batch_put failed: %d/%d keys failed "
|
||||
"(codes=%s, batch_bytes=%d, num_keys=%d), "
|
||||
@@ -1008,6 +1072,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_meta, token_len)
|
||||
save_completed = True
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
@@ -1023,10 +1088,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error("Failed to put key %s, error: %s", keys, e)
|
||||
stored_events.clear()
|
||||
|
||||
if self.enable_kv_event and stored_events:
|
||||
self.update_kv_event(stored_events)
|
||||
finally:
|
||||
if self.enable_kv_event and token_len:
|
||||
self._update_retry_token_ids(
|
||||
req_meta,
|
||||
save_completed,
|
||||
token_ids_start,
|
||||
event_token_ids,
|
||||
)
|
||||
self.finish_store_job(req_meta)
|
||||
self.request_queue.task_done()
|
||||
|
||||
@@ -1274,15 +1347,19 @@ class MooncakeStoreWorker:
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0
|
||||
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"load_async", True
|
||||
kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
assert kv_role is not None
|
||||
self.kv_role = kv_role
|
||||
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
self.can_put = self.kv_role in ("kv_producer", "kv_both") or (
|
||||
extra_config.get("save_decode_cache", False)
|
||||
)
|
||||
self.load_async = extra_config.get("load_async", True)
|
||||
# Mirrors MooncakeStoreConnector._capacity_only.
|
||||
self._capacity_only = self.kv_role == "kv_consumer" and not (
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"enable_lookup", True
|
||||
)
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not extra_config.get("enable_lookup", True)
|
||||
and not self.can_put
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
@@ -1294,11 +1371,6 @@ class MooncakeStoreWorker:
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_config()
|
||||
extra_config = (
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
if vllm_config.kv_transfer_config
|
||||
else {}
|
||||
)
|
||||
self.store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
@@ -1617,7 +1689,7 @@ class MooncakeStoreWorker:
|
||||
db.set_block_len(block_lens)
|
||||
|
||||
# Start transfer threads
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
if self.can_put:
|
||||
ready_event_sending = threading.Event()
|
||||
self.kv_send_thread = KVCacheStoreSendingThread(
|
||||
self.store,
|
||||
@@ -1700,7 +1772,7 @@ class MooncakeStoreWorker:
|
||||
|
||||
assert self.load_async, "load_async must be True for better performance."
|
||||
# Issue stores with CUDA event synchronization.
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
if self.can_put:
|
||||
current_event = None
|
||||
for request in meta.requests:
|
||||
if request.can_save:
|
||||
@@ -1714,8 +1786,6 @@ class MooncakeStoreWorker:
|
||||
request.current_event = current_event
|
||||
assert self.kv_send_thread is not None
|
||||
self.kv_send_thread.add_request(request)
|
||||
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
self._close_ended_store_requests(finished_req_ids, meta)
|
||||
|
||||
# Blocks read by a store job are released by the scheduler when the job
|
||||
|
||||
Reference in New Issue
Block a user