[Mooncake] Only check and store new KV cache range (#46412)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -246,6 +246,20 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
assert masks[1] == [False, False, True, True, False, False, True, True]
|
||||
|
||||
|
||||
def test_store_mask_swa_prefix_stable_as_aligned_length_grows():
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
|
||||
shorter = coord.store_mask(64)[1]
|
||||
longer = coord.store_mask(128)[1]
|
||||
|
||||
assert shorter is not None
|
||||
assert longer is not None
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
|
||||
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
|
||||
@@ -355,6 +369,33 @@ def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
|
||||
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
|
||||
|
||||
|
||||
def test_store_mask_suffix_matches_full_mask_tail():
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
|
||||
full = coord.store_mask(128, num_prompt_tokens=100)
|
||||
suffix = coord.store_mask(128, start_token=64, num_prompt_tokens=100)
|
||||
|
||||
for g_idx, cache_group in enumerate(coord.kv_cache_groups):
|
||||
block_size = cache_group.kv_cache_spec.block_size
|
||||
start_chunk = 64 // block_size
|
||||
end_chunk = 128 // block_size
|
||||
full_mask = full[g_idx]
|
||||
if full_mask is None:
|
||||
assert suffix[g_idx] is None
|
||||
else:
|
||||
assert suffix[g_idx] == full_mask[start_chunk:end_chunk]
|
||||
|
||||
|
||||
def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
|
||||
|
||||
shorter = coord.store_mask(64, num_prompt_tokens=100)[1]
|
||||
longer = coord.store_mask(128, num_prompt_tokens=100)[1]
|
||||
|
||||
assert shorter is not None
|
||||
assert longer is not None
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
# ----- Eagle / MTP interaction with load_mask -----
|
||||
|
||||
|
||||
|
||||
@@ -337,5 +337,5 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
assert out[1][0] == 16 and out[1][1] == 32
|
||||
# Each chunk's hash is its last (4th) fine hash, which already chains the
|
||||
# prior three.
|
||||
assert out[0][2].chunk_hash == fine_hashes[3].hex()
|
||||
assert out[1][2].chunk_hash == fine_hashes[7].hex()
|
||||
assert out[0][2].hex() == fine_hashes[3].hex()
|
||||
assert out[1][2].hex() == fine_hashes[7].hex()
|
||||
|
||||
@@ -36,6 +36,25 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
|
||||
|
||||
class _RecordingBlockHashes:
|
||||
def __init__(self, values: list[bytes]):
|
||||
self.values = values
|
||||
self.accessed: list[int] = []
|
||||
|
||||
def __len__(self):
|
||||
return len(self.values)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx, slice):
|
||||
return [self[i] for i in range(*idx.indices(len(self)))]
|
||||
if idx < 0:
|
||||
idx += len(self.values)
|
||||
if not 0 <= idx < len(self.values):
|
||||
raise IndexError(idx)
|
||||
self.accessed.append(idx)
|
||||
return self.values[idx]
|
||||
|
||||
|
||||
def _default_send_coord() -> mooncake_store_worker.MooncakeStoreCoordinator:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
@@ -53,6 +72,8 @@ def _make_store_sending_thread(
|
||||
coord: mooncake_store_worker.MooncakeStoreCoordinator | None = None,
|
||||
token_databases: list[ChunkedTokenDatabase] | None = None,
|
||||
block_size: int = 16,
|
||||
tp_rank: int = 0,
|
||||
put_step: int = 1,
|
||||
replicate_config: object | None = None,
|
||||
) -> mooncake_store_worker.KVCacheStoreSendingThread:
|
||||
if coord is None:
|
||||
@@ -67,8 +88,8 @@ def _make_store_sending_thread(
|
||||
token_databases=token_databases,
|
||||
block_size=block_size,
|
||||
coord=coord,
|
||||
tp_rank=0,
|
||||
put_step=1,
|
||||
tp_rank=tp_rank,
|
||||
put_step=put_step,
|
||||
kv_role="kv_producer",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=replicate_config,
|
||||
@@ -391,6 +412,248 @@ def test_store_sending_thread_records_mooncake_metrics():
|
||||
assert stats.data["save_put"][0]["status"] == "ok"
|
||||
|
||||
|
||||
def test_process_tokens_uses_mask_num_as_start_chunk():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=96,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [11]
|
||||
assert results == [(64, 96, bytes([11]))]
|
||||
|
||||
|
||||
def test_process_tokens_applies_chunk_mask_before_hash_access():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=128,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
chunk_mask=[False, True],
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [15]
|
||||
assert results == [(96, 128, bytes([15]))]
|
||||
|
||||
|
||||
def test_process_tokens_applies_stride_before_hash_access():
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
block_hashes = _RecordingBlockHashes([bytes([i]) for i in range(16)])
|
||||
|
||||
results = list(
|
||||
db.process_tokens(
|
||||
token_len=128,
|
||||
block_hashes=block_hashes,
|
||||
mask_num=64,
|
||||
put_step=2,
|
||||
put_step_rank=1,
|
||||
)
|
||||
)
|
||||
|
||||
assert block_hashes.accessed == [15]
|
||||
assert results == [(96, 128, bytes([15]))]
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_strides_with_local_phase():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
# Under pressure the request is skipped without persisting anything, so the
|
||||
# saved offset stays at 0.
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=16,
|
||||
block_ids=([0],),
|
||||
block_hashes=[b"a0"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
assert thread._saved_offset.get("req-a", 0) == 0
|
||||
|
||||
thread._store_pressure_active = False
|
||||
thread._skip_store_requests.clear()
|
||||
|
||||
# The next batch resumes from offset 0, re-covering the chunk skipped under
|
||||
# pressure (chunk 0) rather than losing it.
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6130",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6133",
|
||||
]
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[0] == keys
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_masked_chunks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
coord = SimpleNamespace(
|
||||
lcm_block_size=16,
|
||||
store_mask=lambda token_len, start_token, num_prompt_tokens=None: (
|
||||
None,
|
||||
[True, False],
|
||||
),
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=16,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_masked = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
)
|
||||
db_masked.set_kv_caches_base_addr([0x2000])
|
||||
db_masked.set_block_len([256])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_masked],
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
|
||||
masked_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
|
||||
|
||||
assert full_hashes == [b"a2".hex(), b"a3".hex()]
|
||||
assert masked_hashes == [b"a2".hex()]
|
||||
|
||||
|
||||
def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -943,7 +1206,8 @@ def test_worker_put_striding_covers_every_rank_get_namespace(
|
||||
db = w.token_dbs[0]
|
||||
token_len = len(block_hashes) * db.block_size
|
||||
keys = [
|
||||
key.to_string() for _, _, key in db.process_tokens(token_len, block_hashes)
|
||||
PoolKey(db.metadata, block_hash.hex()).to_string()
|
||||
for _, _, block_hash in db.process_tokens(token_len, block_hashes)
|
||||
]
|
||||
assert len(keys) == len(block_hashes)
|
||||
# PUT side: mirrors KVCacheStoreSendingThread's striding slice.
|
||||
@@ -1047,8 +1311,8 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, addrs, sizes: (
|
||||
[256] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
@@ -1113,6 +1377,77 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
assert swa_hashes == {hs[3].hex(), hs[7].hex()}
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = (
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
swa_spec = SlidingWindowSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=None,
|
||||
sliding_window=8,
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], swa_spec)],
|
||||
scheduler_block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([512])
|
||||
db_swa = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=8,
|
||||
hash_block_size=8,
|
||||
)
|
||||
db_swa.set_kv_caches_base_addr([0x2000])
|
||||
db_swa.set_block_len([128])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_swa],
|
||||
block_size=32,
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._saved_offset["r0"] = 32
|
||||
thread._handle_request(
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
full_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:0" in k]
|
||||
swa_hashes = [k.rsplit("@", 1)[-1] for k in keys if "@group:1" in k]
|
||||
assert full_hashes == [hs[7].hex()]
|
||||
assert swa_hashes == [hs[7].hex()]
|
||||
|
||||
|
||||
def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import cast
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
@@ -173,19 +174,21 @@ class MooncakeStoreCoordinator:
|
||||
def store_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int = 0,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group store masks.
|
||||
"""Per-group store masks for the suffix starting at ``start_token``.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
written to the store so a future cache hit can consume it. ``None`` is
|
||||
the all-True sentinel.
|
||||
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
|
||||
``start_token`` should be written to the store so a future cache hit
|
||||
can consume it. ``None`` is the all-True sentinel for the suffix.
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
start_token,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
@@ -202,6 +205,7 @@ class MooncakeStoreCoordinator:
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
0,
|
||||
retention_interval=None,
|
||||
num_prompt_tokens=None,
|
||||
)
|
||||
@@ -209,6 +213,7 @@ class MooncakeStoreCoordinator:
|
||||
def _reachable_masks(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int,
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
@@ -220,20 +225,22 @@ class MooncakeStoreCoordinator:
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
num_chunks = aligned_token_len // spec.block_size
|
||||
end_chunk = aligned_token_len // spec.block_size
|
||||
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=0,
|
||||
end_block=num_chunks,
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=g_idx in self.eagle_group_ids,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == num_chunks
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
masks.append(mask)
|
||||
return tuple(masks)
|
||||
|
||||
|
||||
@@ -180,9 +180,10 @@ class ChunkedTokenDatabase:
|
||||
)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
|
||||
def _make_key_by_hash(self, chunk_hash: str) -> PoolKey:
|
||||
return PoolKey(self.metadata, chunk_hash)
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self.kv_caches_base_addr = kv_caches_base_addr
|
||||
@@ -215,8 +216,17 @@ class ChunkedTokenDatabase:
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
) -> Iterable[tuple[int, int, PoolKey]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
|
||||
*,
|
||||
chunk_mask: list[bool] | None = None,
|
||||
put_step: int = 1,
|
||||
put_step_rank: int = 0,
|
||||
) -> Iterable[tuple[int, int, BlockHash]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
|
||||
|
||||
When there are fewer KV heads than TP ranks, chunks are distributed
|
||||
across TP ranks to avoid duplicate load/store. The assignment keys off
|
||||
the absolute ``chunk_id`` so a given chunk always lands on the same
|
||||
rank regardless of where the processed suffix begins.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens.
|
||||
@@ -224,20 +234,30 @@ class ChunkedTokenDatabase:
|
||||
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
|
||||
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
chunk_mask: Optional mask relative to the first chunk after
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
return
|
||||
chunk_hashes: Iterable[BlockHash] = chunk_hashes_for_block_size(
|
||||
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
|
||||
block_hashes, self.hash_block_size, self.block_size
|
||||
)
|
||||
for chunk_id, h in enumerate(chunk_hashes):
|
||||
start_idx = chunk_id * self.block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
if start_idx < mask_num:
|
||||
start_chunk = max(0, cdiv(mask_num, self.block_size))
|
||||
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
|
||||
if chunk_mask is not None:
|
||||
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
|
||||
for chunk_id in range(start_chunk, max_chunks):
|
||||
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
|
||||
continue
|
||||
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
|
||||
if chunk_id % put_step != put_step_rank:
|
||||
continue
|
||||
h = chunk_hashes[chunk_id]
|
||||
start_idx = chunk_id * self.block_size
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
yield start_idx, end_idx, h
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -473,6 +473,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._store_pressure_active = False
|
||||
self._skip_store_requests: set[str] = set()
|
||||
|
||||
# Per-request high-water mark of tokens actually persisted; the next
|
||||
# batch resumes here, so pressure-skipped or failed ranges are retried.
|
||||
self._saved_offset: dict[str, int] = {}
|
||||
|
||||
def add_stored_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
self.stored_requests[req_id] += 1
|
||||
@@ -487,6 +491,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if req_id in self.stored_requests:
|
||||
del self.stored_requests[req_id]
|
||||
self._skip_store_requests.discard(req_id)
|
||||
self._saved_offset.pop(req_id, None)
|
||||
|
||||
def _record_saved(self, req_id: str, token_len: int) -> None:
|
||||
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self._saved_offset[req_id] = token_len
|
||||
|
||||
def _should_skip_request(self, req_id: str) -> bool:
|
||||
with self.done_task_lock:
|
||||
@@ -526,6 +537,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
try:
|
||||
if token_len == 0:
|
||||
return
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -534,40 +546,42 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
return
|
||||
|
||||
# Resume from where this rank left off; only the new suffix is saved.
|
||||
save_start = self._saved_offset.get(req_id, 0)
|
||||
|
||||
# Within each lcm region only per-spec relevant chunks are loaded
|
||||
# (e.g., SWA or linear attn), so mask out irrelevant chunks
|
||||
store_masks = self.coord.store_mask(
|
||||
token_len, num_prompt_tokens=req_meta.num_prompt_tokens
|
||||
token_len,
|
||||
save_start,
|
||||
num_prompt_tokens=req_meta.num_prompt_tokens,
|
||||
)
|
||||
|
||||
starts: list[int] = []
|
||||
ends: list[int] = []
|
||||
keys: list[str] = []
|
||||
block_hashes: list[BlockHash] = []
|
||||
kv_event_block_hashes: list[BlockHash] = []
|
||||
group_indices: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = store_masks[g_idx]
|
||||
for chunk_idx, (start, end, key) in enumerate(
|
||||
db.process_tokens(token_len, req_meta.block_hashes)
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step_rank = (self.tp_rank + g_idx) % self.put_step
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len,
|
||||
req_meta.block_hashes,
|
||||
mask_num=save_start,
|
||||
chunk_mask=store_masks[g_idx],
|
||||
put_step=self.put_step,
|
||||
put_step_rank=put_step_rank,
|
||||
):
|
||||
if mask is not None and (
|
||||
chunk_idx >= len(mask) or not mask[chunk_idx]
|
||||
):
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(BlockHash(bytes.fromhex(key.chunk_hash)))
|
||||
keys.append(db.key_for(block_hash))
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes.append(block_hash)
|
||||
group_indices.append(g_idx)
|
||||
|
||||
# Apply put_step striding for TP
|
||||
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
|
||||
starts = starts[sl]
|
||||
ends = ends[sl]
|
||||
keys = keys[sl]
|
||||
block_hashes = block_hashes[sl]
|
||||
group_indices = group_indices[sl]
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_id, token_len)
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
@@ -593,13 +607,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
]
|
||||
|
||||
if not missing_indices:
|
||||
self._record_saved(req_id, token_len)
|
||||
return
|
||||
|
||||
starts = [starts[i] for i in missing_indices]
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
block_hashes = [block_hashes[i] for i in missing_indices]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
if len(missing_indices) != len(keys):
|
||||
starts = [starts[i] for i in missing_indices]
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes = [
|
||||
kv_event_block_hashes[i] for i in missing_indices
|
||||
]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
|
||||
logger.debug(
|
||||
"Storing KV cache for %d blocks (groups=%s) for request %s",
|
||||
@@ -612,8 +631,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
|
||||
if self.enable_kv_event:
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [
|
||||
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
|
||||
]
|
||||
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
@@ -687,11 +709,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
"batch succeeds",
|
||||
req_id,
|
||||
)
|
||||
elif self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared after a "
|
||||
"successful store batch"
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_id, token_len)
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
"after a successful store batch"
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
@@ -775,7 +799,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
for start, end, key in db.process_tokens(
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
chunk_idx = start // db.block_size
|
||||
@@ -784,7 +808,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
addr, size, block_id = db.prepare_value(
|
||||
start, end, req_meta.block_ids[g_idx]
|
||||
)
|
||||
key_list.append(key.to_string())
|
||||
key_list.append(db.key_for(block_hash))
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
block_id_list.append(block_id)
|
||||
|
||||
Reference in New Issue
Block a user