diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py index 0cddd56a60..2ad4b79164 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py @@ -246,6 +246,20 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm(): assert masks[1] == [False, False, True, True, False, False, True, True] +def test_store_mask_swa_prefix_stable_as_aligned_length_grows(): + full = _full(32) + swa = _swa(block_size=8, sliding_window=8) + groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)] + coord = _make_coord(groups, hash_block_size=8) + + shorter = coord.store_mask(64)[1] + longer = coord.store_mask(128)[1] + + assert shorter is not None + assert longer is not None + assert longer[: len(shorter)] == shorter + + def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa(): """DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8} and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments @@ -355,6 +369,33 @@ def test_store_mask_retention_interval_keeps_segment_and_replay_tails(): assert masks[1] == [i in (7, 11, 15) for i in range(16)] +def test_store_mask_suffix_matches_full_mask_tail(): + coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64) + full = coord.store_mask(128, num_prompt_tokens=100) + suffix = coord.store_mask(128, start_token=64, num_prompt_tokens=100) + + for g_idx, cache_group in enumerate(coord.kv_cache_groups): + block_size = cache_group.kv_cache_spec.block_size + start_chunk = 64 // block_size + end_chunk = 128 // block_size + full_mask = full[g_idx] + if full_mask is None: + assert suffix[g_idx] is None + else: + assert suffix[g_idx] == full_mask[start_chunk:end_chunk] + + +def test_store_mask_retention_prefix_stable_as_aligned_length_grows(): + coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0) + + shorter = coord.store_mask(64, num_prompt_tokens=100)[1] + longer = coord.store_mask(128, num_prompt_tokens=100)[1] + + assert shorter is not None + assert longer is not None + assert longer[: len(shorter)] == shorter + + # ----- Eagle / MTP interaction with load_mask ----- diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index 9e9a57cdf7..6dcb3c914b 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -337,5 +337,5 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size(): assert out[1][0] == 16 and out[1][1] == 32 # Each chunk's hash is its last (4th) fine hash, which already chains the # prior three. - assert out[0][2].chunk_hash == fine_hashes[3].hex() - assert out[1][2].chunk_hash == fine_hashes[7].hex() + assert out[0][2].hex() == fine_hashes[3].hex() + assert out[1][2].hex() == fine_hashes[7].hex() diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 4231912596..dce582946b 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -36,6 +36,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import from vllm.v1.core.kv_cache_utils import BlockHash +class _RecordingBlockHashes: + def __init__(self, values: list[bytes]): + self.values = values + self.accessed: list[int] = [] + + def __len__(self): + return len(self.values) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return [self[i] for i in range(*idx.indices(len(self)))] + if idx < 0: + idx += len(self.values) + if not 0 <= idx < len(self.values): + raise IndexError(idx) + self.accessed.append(idx) + return self.values[idx] + + def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator: from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec @@ -53,6 +72,8 @@ def _make_store_sending_thread( coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None, token_databases: list[ChunkedTokenDatabase] | None = None, block_size: int = 16, + tp_rank: int = 0, + put_step: int = 1, replicate_config: object | None = None, ) -> mooncake_store_worker.KVCacheStoreSendingThread: if coord is None: @@ -67,8 +88,8 @@ def _make_store_sending_thread( token_databases=token_databases, block_size=block_size, coord=coord, - tp_rank=0, - put_step=1, + tp_rank=tp_rank, + put_step=put_step, kv_role="kv_producer", ready_event=threading.Event(), replicate_config=replicate_config, @@ -391,6 +412,248 @@ def test_store_sending_thread_records_mooncake_metrics(): assert stats.data["save_put"][0]["status"] == "ok" +def test_process_tokens_uses_mask_num_as_start_chunk(): + db = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0), + block_size=32, + hash_block_size=8, + ) + block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)]) + + results = list( + db.process_tokens( + token_len=96, + block_hashes=block_hashes, + mask_num=64, + ) + ) + + assert block_hashes.accessed == [11] + assert results == [(64, 96, bytes([11]))] + + +def test_process_tokens_applies_chunk_mask_before_hash_access(): + db = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0), + block_size=32, + hash_block_size=8, + ) + block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)]) + + results = list( + db.process_tokens( + token_len=128, + block_hashes=block_hashes, + mask_num=64, + chunk_mask=[False, True], + ) + ) + + assert block_hashes.accessed == [15] + assert results == [(96, 128, bytes([15]))] + + +def test_process_tokens_applies_stride_before_hash_access(): + db = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0), + block_size=32, + hash_block_size=8, + ) + block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)]) + + results = list( + db.process_tokens( + token_len=128, + block_hashes=block_hashes, + mask_num=64, + put_step=2, + put_step_rank=1, + ) + ) + + assert block_hashes.accessed == [15] + assert results == [(96, 128, bytes([15]))] + + +def test_store_sending_thread_delta_saves_only_new_full_attention_chunks(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.return_value = [256, 256] + thread = _make_store_sending_thread(store) + + thread.add_stored_request("req-a") + thread._saved_offset["req-a"] = 32 + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3],), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + assert keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133", + ] + assert store.batch_put_from_multi_buffers.call_args.args[0] == keys + + +def test_store_sending_thread_delta_strides_with_local_phase(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.return_value = [256] + thread = _make_store_sending_thread(store, tp_rank=0, put_step=2) + + thread.add_stored_request("req-a") + thread._saved_offset["req-a"] = 16 + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3],), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + assert keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", + ] + assert store.batch_put_from_multi_buffers.call_args.args[0] == keys + + +def test_store_sending_thread_retries_skipped_range_after_pressure(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys) + thread = _make_store_sending_thread(store, tp_rank=0, put_step=2) + + # Under pressure the request is skipped without persisting anything, so the + # saved offset stays at 0. + thread._store_pressure_active = True + thread._skip_store_requests.add("req-a") + + thread.add_stored_request("req-a") + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=16, + block_ids=([0],), + block_hashes=[b"a0"], + can_save=True, + ) + ) + + store.batch_is_exist.assert_not_called() + assert thread._saved_offset.get("req-a", 0) == 0 + + thread._store_pressure_active = False + thread._skip_store_requests.clear() + + # The next batch resumes from offset 0, re-covering the chunk skipped under + # pressure (chunk 0) rather than losing it. + thread.add_stored_request("req-a") + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3],), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + assert keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132", + ] + assert store.batch_put_from_multi_buffers.call_args.args[0] == keys + + +def test_store_sending_thread_delta_start_rank_saves_second_local_chunk(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.return_value = [256, 256] + thread = _make_store_sending_thread(store, tp_rank=1, put_step=2) + + thread.add_stored_request("req-a") + thread._saved_offset["req-a"] = 16 + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3],), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + assert keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133", + ] + assert store.batch_put_from_multi_buffers.call_args.args[0] == keys + + +def test_store_sending_thread_delta_saves_only_new_masked_chunks(): + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.side_effect = ( + lambda keys, addrs, sizes, replicate_config: [256] * len(keys) + ) + coord = SimpleNamespace( + lcm_block_size=16, + store_mask=lambda token_len, start_token, num_prompt_tokens=None: ( + None, + [True, False], + ), + ) + + db_full = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=16, + ) + db_full.set_kv_caches_base_addr([0x1000]) + db_full.set_block_len([256]) + db_masked = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), + block_size=16, + ) + db_masked.set_kv_caches_base_addr([0x2000]) + db_masked.set_block_len([256]) + + thread = _make_store_sending_thread( + store, + coord=coord, + token_databases=[db_full, db_masked], + ) + + thread.add_stored_request("req-a") + thread._saved_offset["req-a"] = 32 + thread._handle_request( + ReqMeta( + req_id="req-a", + token_len_chunk=64, + block_ids=([0, 1, 2, 3], [0, 1, 2, 3]), + block_hashes=[b"a0", b"a1", b"a2", b"a3"], + can_save=True, + ) + ) + + keys = store.batch_is_exist.call_args.args[0] + full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k] + masked_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k] + + assert full_hashes == [b"a2".hex(), b"a3".hex()] + assert masked_hashes == [b"a2".hex()] + + def test_store_sending_thread_only_skips_on_no_available_handle(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) @@ -943,7 +1206,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace( db = w.token_dbs[0] token_len = len(block_hashes) * db.block_size keys = [ - key.to_string() for _, _, key in db.process_tokens(token_len, block_hashes) + PoolKey(db.metadata, block_hash.hex()).to_string() + for _, _, block_hash in db.process_tokens(token_len, block_hashes) ] assert len(keys) == len(block_hashes) # PUT side: mirrors KVCacheStoreSendingThread's striding slice. @@ -1047,8 +1311,8 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) - store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: ( - [256] * len(keys) + store.batch_put_from_multi_buffers.side_effect = ( + lambda keys, addrs, sizes, replicate_config: [256] * len(keys) ) full_spec = FullAttentionSpec( @@ -1113,6 +1377,77 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window(): assert swa_hashes == {hs[3].hex(), hs[7].hex()} +def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks(): + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + SlidingWindowSpec, + ) + + store = MagicMock() + store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) + store.batch_put_from_multi_buffers.side_effect = ( + lambda keys, addrs, sizes, replicate_config: [256] * len(keys) + ) + + full_spec = FullAttentionSpec( + block_size=32, num_kv_heads=8, head_size=64, dtype=None + ) + swa_spec = SlidingWindowSpec( + block_size=8, + num_kv_heads=8, + head_size=64, + dtype=None, + sliding_window=8, + ) + coord = mooncake_store_worker.MooncakeStoreCoordinator( + [KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)], + scheduler_block_size=32, + hash_block_size=8, + ) + + db_full = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=32, + hash_block_size=8, + ) + db_full.set_kv_caches_base_addr([0x1000]) + db_full.set_block_len([512]) + db_swa = ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), + block_size=8, + hash_block_size=8, + ) + db_swa.set_kv_caches_base_addr([0x2000]) + db_swa.set_block_len([128]) + + thread = _make_store_sending_thread( + store, + coord=coord, + token_databases=[db_full, db_swa], + block_size=32, + ) + + hs = [bytes([i + 1]) * 4 for i in range(8)] + thread.add_stored_request("r0") + thread._saved_offset["r0"] = 32 + thread._handle_request( + ReqMeta( + req_id="r0", + token_len_chunk=64, + block_ids=([0, 1], list(range(8))), + block_hashes=hs, + can_save=True, + ) + ) + + keys = store.batch_put_from_multi_buffers.call_args.args[0] + full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k] + swa_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k] + assert full_hashes == [hs[7].hex()] + assert swa_hashes == [hs[7].hex()] + + def test_store_sending_thread_kv_events_use_group_chunk_metadata(): from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash from vllm.v1.kv_cache_interface import ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index 89ffb56003..6923ceb24d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -8,6 +8,7 @@ from typing import cast from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( chunk_hashes_for_block_size, ) +from vllm.utils.math_utils import cdiv from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_utils import ( BlockHash, @@ -173,19 +174,21 @@ class MooncakeStoreCoordinator: def store_mask( self, aligned_token_len: int, + start_token: int = 0, num_prompt_tokens: int | None = None, ) -> tuple[list[bool] | None, ...]: - """Per-group store masks. + """Per-group store masks for the suffix starting at ``start_token``. - ``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be - written to the store so a future cache hit can consume it. ``None`` is - the all-True sentinel. + ``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after* + ``start_token`` should be written to the store so a future cache hit + can consume it. ``None`` is the all-True sentinel for the suffix. Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask`` so the store retains exactly the blocks the local prefix cache would. """ return self._reachable_masks( aligned_token_len, + start_token, retention_interval=self.retention_interval, num_prompt_tokens=num_prompt_tokens, ) @@ -202,6 +205,7 @@ class MooncakeStoreCoordinator: """ return self._reachable_masks( aligned_token_len, + 0, retention_interval=None, num_prompt_tokens=None, ) @@ -209,6 +213,7 @@ class MooncakeStoreCoordinator: def _reachable_masks( self, aligned_token_len: int, + start_token: int, *, retention_interval: int | None, num_prompt_tokens: int | None, @@ -220,20 +225,22 @@ class MooncakeStoreCoordinator: masks: list[list[bool] | None] = [] for g_idx, g in enumerate(self.kv_cache_groups): spec = _unwrap_spec(g.kv_cache_spec) - num_chunks = aligned_token_len // spec.block_size + end_chunk = aligned_token_len // spec.block_size + start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size))) manager_cls = KVCacheSpecRegistry.get_manager_class(spec) assert manager_cls is not None + use_eagle = g_idx in self.eagle_group_ids mask = manager_cls.reachable_block_mask( - start_block=0, - end_block=num_chunks, + start_block=start_chunk, + end_block=end_chunk, alignment_tokens=self.lcm_block_size, kv_cache_spec=spec, - use_eagle=g_idx in self.eagle_group_ids, + use_eagle=use_eagle, retention_interval=retention_interval, num_prompt_tokens=num_prompt_tokens, ) if mask is not None: - assert len(mask) == num_chunks + assert len(mask) == end_chunk - start_chunk masks.append(mask) return tuple(masks) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index aa9fc38f86..7f8168ab38 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -180,9 +180,10 @@ class ChunkedTokenDatabase: ) self.kv_caches_base_addr: list[int] = [] self.block_len: list[int] = [] + self._key_prefix = PoolKey.build_prefix(metadata) - def _make_key_by_hash(self, chunk_hash: str) -> PoolKey: - return PoolKey(self.metadata, chunk_hash) + def key_for(self, chunk_hash: BlockHash) -> str: + return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex()) def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]): self.kv_caches_base_addr = kv_caches_base_addr @@ -215,8 +216,17 @@ class ChunkedTokenDatabase: token_len: int, block_hashes: list[BlockHash], mask_num: int = 0, - ) -> Iterable[tuple[int, int, PoolKey]]: - """Process tokens and yield (start_idx, end_idx, pool_key) tuples. + *, + chunk_mask: list[bool] | None = None, + put_step: int = 1, + put_step_rank: int = 0, + ) -> Iterable[tuple[int, int, BlockHash]]: + """Process tokens and yield (start_idx, end_idx, block_hash) tuples. + + When there are fewer KV heads than TP ranks, chunks are distributed + across TP ranks to avoid duplicate load/store. The assignment keys off + the absolute ``chunk_id`` so a given chunk always lands on the same + rank regardless of where the processed suffix begins. Args: token_len: Total number of tokens. @@ -224,20 +234,30 @@ class ChunkedTokenDatabase: When ``block_size > hash_block_size`` each group's ``block_size`` chunk is keyed by its last sub-hash via ``chunk_hashes_for_block_size``. mask_num: Number of tokens to skip from the beginning. + chunk_mask: Optional mask relative to the first chunk after + ``mask_num``. False entries are skipped before hash access. + put_step: Stride for distributing chunks across ranks. + put_step_rank: ``chunk_id % put_step`` value this rank stores. """ + assert put_step > 0 if not block_hashes: return - chunk_hashes: Iterable[BlockHash] = chunk_hashes_for_block_size( + chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size( block_hashes, self.hash_block_size, self.block_size ) - for chunk_id, h in enumerate(chunk_hashes): - start_idx = chunk_id * self.block_size - if start_idx >= token_len: - break - end_idx = min(start_idx + self.block_size, token_len) - if start_idx < mask_num: + start_chunk = max(0, cdiv(mask_num, self.block_size)) + max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size)) + if chunk_mask is not None: + max_chunks = min(max_chunks, start_chunk + len(chunk_mask)) + for chunk_id in range(start_chunk, max_chunks): + if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]: continue - yield start_idx, end_idx, self._make_key_by_hash(h.hex()) + if chunk_id % put_step != put_step_rank: + continue + h = chunk_hashes[chunk_id] + start_idx = chunk_id * self.block_size + end_idx = min(start_idx + self.block_size, token_len) + yield start_idx, end_idx, h @dataclass diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index c78226c410..127b8e4d4b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -473,6 +473,10 @@ class KVCacheStoreSendingThread(KVTransferThread): self._store_pressure_active = False self._skip_store_requests: set[str] = set() + # Per-request high-water mark of tokens actually persisted; the next + # batch resumes here, so pressure-skipped or failed ranges are retried. + self._saved_offset: dict[str, int] = {} + def add_stored_request(self, req_id: str): with self.done_task_lock: self.stored_requests[req_id] += 1 @@ -487,6 +491,13 @@ class KVCacheStoreSendingThread(KVTransferThread): if req_id in self.stored_requests: del self.stored_requests[req_id] self._skip_store_requests.discard(req_id) + self._saved_offset.pop(req_id, None) + + def _record_saved(self, req_id: str, token_len: int) -> None: + # Guard on liveness so a concurrent finish/preempt pop isn't recreated. + with self.done_task_lock: + if req_id in self.stored_requests: + self._saved_offset[req_id] = token_len def _should_skip_request(self, req_id: str) -> bool: with self.done_task_lock: @@ -526,6 +537,7 @@ class KVCacheStoreSendingThread(KVTransferThread): try: if token_len == 0: return + if self._should_skip_request(req_id): logger.debug( "Skipping Mooncake store for request %s while CPU/disk " @@ -534,40 +546,42 @@ class KVCacheStoreSendingThread(KVTransferThread): ) return + # Resume from where this rank left off; only the new suffix is saved. + save_start = self._saved_offset.get(req_id, 0) + # Within each lcm region only per-spec relevant chunks are loaded # (e.g., SWA or linear attn), so mask out irrelevant chunks store_masks = self.coord.store_mask( - token_len, num_prompt_tokens=req_meta.num_prompt_tokens + token_len, + save_start, + num_prompt_tokens=req_meta.num_prompt_tokens, ) + starts: list[int] = [] ends: list[int] = [] keys: list[str] = [] - block_hashes: list[BlockHash] = [] + kv_event_block_hashes: list[BlockHash] = [] group_indices: list[int] = [] for g_idx, db in enumerate(self.token_databases): - mask = store_masks[g_idx] - for chunk_idx, (start, end, key) in enumerate( - db.process_tokens(token_len, req_meta.block_hashes) + # Rotate the stride phase per group to balance load across ranks. + put_step_rank = (self.tp_rank + g_idx) % self.put_step + for start, end, block_hash in db.process_tokens( + token_len, + req_meta.block_hashes, + mask_num=save_start, + chunk_mask=store_masks[g_idx], + put_step=self.put_step, + put_step_rank=put_step_rank, ): - if mask is not None and ( - chunk_idx >= len(mask) or not mask[chunk_idx] - ): - continue starts.append(start) ends.append(end) - keys.append(key.to_string()) - block_hashes.append(BlockHash(bytes.fromhex(key.chunk_hash))) + keys.append(db.key_for(block_hash)) + if self.enable_kv_event: + kv_event_block_hashes.append(block_hash) group_indices.append(g_idx) - # Apply put_step striding for TP - sl = slice(self.tp_rank % self.put_step, None, self.put_step) - starts = starts[sl] - ends = ends[sl] - keys = keys[sl] - block_hashes = block_hashes[sl] - group_indices = group_indices[sl] - if not keys: + self._record_saved(req_id, token_len) return # Check which blocks already exist (dedup) @@ -593,13 +607,18 @@ class KVCacheStoreSendingThread(KVTransferThread): ] if not missing_indices: + self._record_saved(req_id, token_len) return - starts = [starts[i] for i in missing_indices] - ends = [ends[i] for i in missing_indices] - keys = [keys[i] for i in missing_indices] - block_hashes = [block_hashes[i] for i in missing_indices] - group_indices = [group_indices[i] for i in missing_indices] + if len(missing_indices) != len(keys): + starts = [starts[i] for i in missing_indices] + ends = [ends[i] for i in missing_indices] + keys = [keys[i] for i in missing_indices] + if self.enable_kv_event: + kv_event_block_hashes = [ + kv_event_block_hashes[i] for i in missing_indices + ] + group_indices = [group_indices[i] for i in missing_indices] logger.debug( "Storing KV cache for %d blocks (groups=%s) for request %s", @@ -612,8 +631,11 @@ class KVCacheStoreSendingThread(KVTransferThread): sizes: list[list[int]] = [] stored_events: list[BlockStored] = [] # parent_block_hash chains live within a group, not across. - prev_key_per_group: dict[int, Any] = {} - new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes] + if self.enable_kv_event: + prev_key_per_group: dict[int, Any] = {} + new_block_hashes = [ + maybe_convert_block_hash(bh) for bh in kv_event_block_hashes + ] for idx, (s, e, g_idx) in enumerate( zip(starts, ends, group_indices, strict=True) @@ -687,11 +709,13 @@ class KVCacheStoreSendingThread(KVTransferThread): "batch succeeds", req_id, ) - elif self._clear_store_pressure(): - logger.info( - "Mooncake CPU/disk offloading pressure cleared after a " - "successful store batch" - ) + else: + self._record_saved(req_id, token_len) + if self._clear_store_pressure(): + logger.info( + "Mooncake CPU/disk offloading pressure cleared " + "after a successful store batch" + ) except Exception as e: self._record_operation( "save_put", @@ -775,7 +799,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): block_id_list: list[int] = [] for g_idx, db in enumerate(self.token_databases): mask = load_mask_per_group[g_idx] - for start, end, key in db.process_tokens( + for start, end, block_hash in db.process_tokens( token_len, req_meta.block_hashes, mask_num ): chunk_idx = start // db.block_size @@ -784,7 +808,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): addr, size, block_id = db.prepare_value( start, end, req_meta.block_ids[g_idx] ) - key_list.append(key.to_string()) + key_list.append(db.key_for(block_hash)) addr_list.append(addr) size_list.append(size) block_id_list.append(block_id)