[Mooncake] Only check and store new KV cache range (#46412)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Wei Zhao
2026-06-24 03:10:50 -07:00
committed by GitHub
co-authored by Yifan Qiao
parent ede54b926e
commit d20dbf921b
6 changed files with 488 additions and 61 deletions
@@ -246,6 +246,20 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
assert masks[1] == [False, False, True, True, False, False, True, True]
def test_store_mask_swa_prefix_stable_as_aligned_length_grows():
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
shorter = coord.store_mask(64)[1]
longer = coord.store_mask(128)[1]
assert shorter is not None
assert longer is not None
assert longer[: len(shorter)] == shorter
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
@@ -355,6 +369,33 @@ def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
def test_store_mask_suffix_matches_full_mask_tail():
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
full = coord.store_mask(128, num_prompt_tokens=100)
suffix = coord.store_mask(128, start_token=64, num_prompt_tokens=100)
for g_idx, cache_group in enumerate(coord.kv_cache_groups):
block_size = cache_group.kv_cache_spec.block_size
start_chunk = 64 // block_size
end_chunk = 128 // block_size
full_mask = full[g_idx]
if full_mask is None:
assert suffix[g_idx] is None
else:
assert suffix[g_idx] == full_mask[start_chunk:end_chunk]
def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
shorter = coord.store_mask(64, num_prompt_tokens=100)[1]
longer = coord.store_mask(128, num_prompt_tokens=100)[1]
assert shorter is not None
assert longer is not None
assert longer[: len(shorter)] == shorter
# ----- Eagle / MTP interaction with load_mask -----
@@ -337,5 +337,5 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
assert out[1][0] == 16 and out[1][1] == 32
# Each chunk's hash is its last (4th) fine hash, which already chains the
# prior three.
assert out[0][2].chunk_hash == fine_hashes[3].hex()
assert out[1][2].chunk_hash == fine_hashes[7].hex()
assert out[0][2].hex() == fine_hashes[3].hex()
assert out[1][2].hex() == fine_hashes[7].hex()
@@ -36,6 +36,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
from vllm.v1.core.kv_cache_utils import BlockHash
class _RecordingBlockHashes:
def __init__(self, values: list[bytes]):
self.values = values
self.accessed: list[int] = []
def __len__(self):
return len(self.values)
def __getitem__(self, idx):
if isinstance(idx, slice):
return [self[i] for i in range(*idx.indices(len(self)))]
if idx < 0:
idx += len(self.values)
if not 0 <= idx < len(self.values):
raise IndexError(idx)
self.accessed.append(idx)
return self.values[idx]
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
@@ -53,6 +72,8 @@ def _make_store_sending_thread(
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
token_databases: list[ChunkedTokenDatabase] | None = None,
block_size: int = 16,
tp_rank: int = 0,
put_step: int = 1,
replicate_config: object | None = None,
) -> mooncake_store_worker.KVCacheStoreSendingThread:
if coord is None:
@@ -67,8 +88,8 @@ def _make_store_sending_thread(
token_databases=token_databases,
block_size=block_size,
coord=coord,
tp_rank=0,
put_step=1,
tp_rank=tp_rank,
put_step=put_step,
kv_role="kv_producer",
ready_event=threading.Event(),
replicate_config=replicate_config,
@@ -391,6 +412,248 @@ def test_store_sending_thread_records_mooncake_metrics():
assert stats.data["save_put"][0]["status"] == "ok"
def test_process_tokens_uses_mask_num_as_start_chunk():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=96,
block_hashes=block_hashes,
mask_num=64,
)
)
assert block_hashes.accessed == [11]
assert results == [(64, 96, bytes([11]))]
def test_process_tokens_applies_chunk_mask_before_hash_access():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=128,
block_hashes=block_hashes,
mask_num=64,
chunk_mask=[False, True],
)
)
assert block_hashes.accessed == [15]
assert results == [(96, 128, bytes([15]))]
def test_process_tokens_applies_stride_before_hash_access():
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
block_size=32,
hash_block_size=8,
)
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
results = list(
db.process_tokens(
token_len=128,
block_hashes=block_hashes,
mask_num=64,
put_step=2,
put_step_rank=1,
)
)
assert block_hashes.accessed == [15]
assert results == [(96, 128, bytes([15]))]
def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_strides_with_local_phase():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256]
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_retries_skipped_range_after_pressure():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
# Under pressure the request is skipped without persisting anything, so the
# saved offset stays at 0.
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=16,
block_ids=([0],),
block_hashes=[b"a0"],
can_save=True,
)
)
store.batch_is_exist.assert_not_called()
assert thread._saved_offset.get("req-a", 0) == 0
thread._store_pressure_active = False
thread._skip_store_requests.clear()
# The next batch resumes from offset 0, re-covering the chunk skipped under
# pressure (chunk 0) rather than losing it.
thread.add_stored_request("req-a")
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
]
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
def test_store_sending_thread_delta_saves_only_new_masked_chunks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
coord = SimpleNamespace(
lcm_block_size=16,
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
None,
[True, False],
),
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=16,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_masked = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
)
db_masked.set_kv_caches_base_addr([0x2000])
db_masked.set_block_len([256])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_masked],
)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
)
keys = store.batch_is_exist.call_args.args[0]
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
masked_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
assert full_hashes == [b"a2".hex(), b"a3".hex()]
assert masked_hashes == [b"a2".hex()]
def test_store_sending_thread_only_skips_on_no_available_handle():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -943,7 +1206,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
db = w.token_dbs[0]
token_len = len(block_hashes) * db.block_size
keys = [
key.to_string() for _, _, key in db.process_tokens(token_len, block_hashes)
PoolKey(db.metadata, block_hash.hex()).to_string()
for _, _, block_hash in db.process_tokens(token_len, block_hashes)
]
assert len(keys) == len(block_hashes)
# PUT side: mirrors KVCacheStoreSendingThread's striding slice.
@@ -1047,8 +1311,8 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
[256] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
full_spec = FullAttentionSpec(
@@ -1113,6 +1377,77 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
full_spec = FullAttentionSpec(
block_size=32, num_kv_heads=8, head_size=64, dtype=None
)
swa_spec = SlidingWindowSpec(
block_size=8,
num_kv_heads=8,
head_size=64,
dtype=None,
sliding_window=8,
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
scheduler_block_size=32,
hash_block_size=8,
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=32,
hash_block_size=8,
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([512])
db_swa = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=8,
)
db_swa.set_kv_caches_base_addr([0x2000])
db_swa.set_block_len([128])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_swa],
block_size=32,
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._saved_offset["r0"] = 32
thread._handle_request(
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
swa_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
assert full_hashes == [hs[7].hex()]
assert swa_hashes == [hs[7].hex()]
def test_store_sending_thread_kv_events_use_group_chunk_metadata():
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
@@ -8,6 +8,7 @@ from typing import cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
@@ -173,19 +174,21 @@ class MooncakeStoreCoordinator:
def store_mask(
self,
aligned_token_len: int,
start_token: int = 0,
num_prompt_tokens: int | None = None,
) -> tuple[list[bool] | None, ...]:
"""Per-group store masks.
"""Per-group store masks for the suffix starting at ``start_token``.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
written to the store so a future cache hit can consume it. ``None`` is
the all-True sentinel.
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
``start_token`` should be written to the store so a future cache hit
can consume it. ``None`` is the all-True sentinel for the suffix.
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
"""
return self._reachable_masks(
aligned_token_len,
start_token,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
)
@@ -202,6 +205,7 @@ class MooncakeStoreCoordinator:
"""
return self._reachable_masks(
aligned_token_len,
0,
retention_interval=None,
num_prompt_tokens=None,
)
@@ -209,6 +213,7 @@ class MooncakeStoreCoordinator:
def _reachable_masks(
self,
aligned_token_len: int,
start_token: int,
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
@@ -220,20 +225,22 @@ class MooncakeStoreCoordinator:
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
num_chunks = aligned_token_len // spec.block_size
end_chunk = aligned_token_len // spec.block_size
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
mask = manager_cls.reachable_block_mask(
start_block=0,
end_block=num_chunks,
start_block=start_chunk,
end_block=end_chunk,
alignment_tokens=self.lcm_block_size,
kv_cache_spec=spec,
use_eagle=g_idx in self.eagle_group_ids,
use_eagle=use_eagle,
retention_interval=retention_interval,
num_prompt_tokens=num_prompt_tokens,
)
if mask is not None:
assert len(mask) == num_chunks
assert len(mask) == end_chunk - start_chunk
masks.append(mask)
return tuple(masks)
@@ -180,9 +180,10 @@ class ChunkedTokenDatabase:
)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
self._key_prefix = PoolKey.build_prefix(metadata)
def _make_key_by_hash(self, chunk_hash: str) -> PoolKey:
return PoolKey(self.metadata, chunk_hash)
def key_for(self, chunk_hash: BlockHash) -> str:
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
self.kv_caches_base_addr = kv_caches_base_addr
@@ -215,8 +216,17 @@ class ChunkedTokenDatabase:
token_len: int,
block_hashes: list[BlockHash],
mask_num: int = 0,
) -> Iterable[tuple[int, int, PoolKey]]:
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
*,
chunk_mask: list[bool] | None = None,
put_step: int = 1,
put_step_rank: int = 0,
) -> Iterable[tuple[int, int, BlockHash]]:
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
When there are fewer KV heads than TP ranks, chunks are distributed
across TP ranks to avoid duplicate load/store. The assignment keys off
the absolute ``chunk_id`` so a given chunk always lands on the same
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens.
@@ -224,20 +234,30 @@ class ChunkedTokenDatabase:
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
mask_num: Number of tokens to skip from the beginning.
chunk_mask: Optional mask relative to the first chunk after
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
return
chunk_hashes: Iterable[BlockHash] = chunk_hashes_for_block_size(
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, self.block_size
)
for chunk_id, h in enumerate(chunk_hashes):
start_idx = chunk_id * self.block_size
if start_idx >= token_len:
break
end_idx = min(start_idx + self.block_size, token_len)
if start_idx < mask_num:
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
continue
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
if chunk_id % put_step != put_step_rank:
continue
h = chunk_hashes[chunk_id]
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
yield start_idx, end_idx, h
@dataclass
@@ -473,6 +473,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._store_pressure_active = False
self._skip_store_requests: set[str] = set()
# Per-request high-water mark of tokens actually persisted; the next
# batch resumes here, so pressure-skipped or failed ranges are retried.
self._saved_offset: dict[str, int] = {}
def add_stored_request(self, req_id: str):
with self.done_task_lock:
self.stored_requests[req_id] += 1
@@ -487,6 +491,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
if req_id in self.stored_requests:
del self.stored_requests[req_id]
self._skip_store_requests.discard(req_id)
self._saved_offset.pop(req_id, None)
def _record_saved(self, req_id: str, token_len: int) -> None:
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
with self.done_task_lock:
if req_id in self.stored_requests:
self._saved_offset[req_id] = token_len
def _should_skip_request(self, req_id: str) -> bool:
with self.done_task_lock:
@@ -526,6 +537,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
try:
if token_len == 0:
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -534,40 +546,42 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
return
# Resume from where this rank left off; only the new suffix is saved.
save_start = self._saved_offset.get(req_id, 0)
# Within each lcm region only per-spec relevant chunks are loaded
# (e.g., SWA or linear attn), so mask out irrelevant chunks
store_masks = self.coord.store_mask(
token_len, num_prompt_tokens=req_meta.num_prompt_tokens
token_len,
save_start,
num_prompt_tokens=req_meta.num_prompt_tokens,
)
starts: list[int] = []
ends: list[int] = []
keys: list[str] = []
block_hashes: list[BlockHash] = []
kv_event_block_hashes: list[BlockHash] = []
group_indices: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = store_masks[g_idx]
for chunk_idx, (start, end, key) in enumerate(
db.process_tokens(token_len, req_meta.block_hashes)
# Rotate the stride phase per group to balance load across ranks.
put_step_rank = (self.tp_rank + g_idx) % self.put_step
for start, end, block_hash in db.process_tokens(
token_len,
req_meta.block_hashes,
mask_num=save_start,
chunk_mask=store_masks[g_idx],
put_step=self.put_step,
put_step_rank=put_step_rank,
):
if mask is not None and (
chunk_idx >= len(mask) or not mask[chunk_idx]
):
continue
starts.append(start)
ends.append(end)
keys.append(key.to_string())
block_hashes.append(BlockHash(bytes.fromhex(key.chunk_hash)))
keys.append(db.key_for(block_hash))
if self.enable_kv_event:
kv_event_block_hashes.append(block_hash)
group_indices.append(g_idx)
# Apply put_step striding for TP
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
starts = starts[sl]
ends = ends[sl]
keys = keys[sl]
block_hashes = block_hashes[sl]
group_indices = group_indices[sl]
if not keys:
self._record_saved(req_id, token_len)
return
# Check which blocks already exist (dedup)
@@ -593,13 +607,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
]
if not missing_indices:
self._record_saved(req_id, token_len)
return
starts = [starts[i] for i in missing_indices]
ends = [ends[i] for i in missing_indices]
keys = [keys[i] for i in missing_indices]
block_hashes = [block_hashes[i] for i in missing_indices]
group_indices = [group_indices[i] for i in missing_indices]
if len(missing_indices) != len(keys):
starts = [starts[i] for i in missing_indices]
ends = [ends[i] for i in missing_indices]
keys = [keys[i] for i in missing_indices]
if self.enable_kv_event:
kv_event_block_hashes = [
kv_event_block_hashes[i] for i in missing_indices
]
group_indices = [group_indices[i] for i in missing_indices]
logger.debug(
"Storing KV cache for %d blocks (groups=%s) for request %s",
@@ -612,8 +631,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
sizes: list[list[int]] = []
stored_events: list[BlockStored] = []
# parent_block_hash chains live within a group, not across.
prev_key_per_group: dict[int, Any] = {}
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
if self.enable_kv_event:
prev_key_per_group: dict[int, Any] = {}
new_block_hashes = [
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
]
for idx, (s, e, g_idx) in enumerate(
zip(starts, ends, group_indices, strict=True)
@@ -687,11 +709,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
"batch succeeds",
req_id,
)
elif self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared after a "
"successful store batch"
)
else:
self._record_saved(req_id, token_len)
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared "
"after a successful store batch"
)
except Exception as e:
self._record_operation(
"save_put",
@@ -775,7 +799,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
block_id_list: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = load_mask_per_group[g_idx]
for start, end, key in db.process_tokens(
for start, end, block_hash in db.process_tokens(
token_len, req_meta.block_hashes, mask_num
):
chunk_idx = start // db.block_size
@@ -784,7 +808,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
addr, size, block_id = db.prepare_value(
start, end, req_meta.block_ids[g_idx]
)
key_list.append(key.to_string())
key_list.append(db.key_for(block_hash))
addr_list.append(addr)
size_list.append(size)
block_id_list.append(block_id)