[Mooncake] Only check and store new KV cache range (#46412)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -8,6 +8,7 @@ from typing import cast
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
@@ -173,19 +174,21 @@ class MooncakeStoreCoordinator:
|
||||
def store_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int = 0,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group store masks.
|
||||
"""Per-group store masks for the suffix starting at ``start_token``.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
written to the store so a future cache hit can consume it. ``None`` is
|
||||
the all-True sentinel.
|
||||
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
|
||||
``start_token`` should be written to the store so a future cache hit
|
||||
can consume it. ``None`` is the all-True sentinel for the suffix.
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
start_token,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
@@ -202,6 +205,7 @@ class MooncakeStoreCoordinator:
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
0,
|
||||
retention_interval=None,
|
||||
num_prompt_tokens=None,
|
||||
)
|
||||
@@ -209,6 +213,7 @@ class MooncakeStoreCoordinator:
|
||||
def _reachable_masks(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int,
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
@@ -220,20 +225,22 @@ class MooncakeStoreCoordinator:
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
num_chunks = aligned_token_len // spec.block_size
|
||||
end_chunk = aligned_token_len // spec.block_size
|
||||
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=0,
|
||||
end_block=num_chunks,
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=g_idx in self.eagle_group_ids,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == num_chunks
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
masks.append(mask)
|
||||
return tuple(masks)
|
||||
|
||||
|
||||
@@ -180,9 +180,10 @@ class ChunkedTokenDatabase:
|
||||
)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
|
||||
def _make_key_by_hash(self, chunk_hash: str) -> PoolKey:
|
||||
return PoolKey(self.metadata, chunk_hash)
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self.kv_caches_base_addr = kv_caches_base_addr
|
||||
@@ -215,8 +216,17 @@ class ChunkedTokenDatabase:
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
) -> Iterable[tuple[int, int, PoolKey]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, pool_key) tuples.
|
||||
*,
|
||||
chunk_mask: list[bool] | None = None,
|
||||
put_step: int = 1,
|
||||
put_step_rank: int = 0,
|
||||
) -> Iterable[tuple[int, int, BlockHash]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
|
||||
|
||||
When there are fewer KV heads than TP ranks, chunks are distributed
|
||||
across TP ranks to avoid duplicate load/store. The assignment keys off
|
||||
the absolute ``chunk_id`` so a given chunk always lands on the same
|
||||
rank regardless of where the processed suffix begins.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens.
|
||||
@@ -224,20 +234,30 @@ class ChunkedTokenDatabase:
|
||||
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
|
||||
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
chunk_mask: Optional mask relative to the first chunk after
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
return
|
||||
chunk_hashes: Iterable[BlockHash] = chunk_hashes_for_block_size(
|
||||
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
|
||||
block_hashes, self.hash_block_size, self.block_size
|
||||
)
|
||||
for chunk_id, h in enumerate(chunk_hashes):
|
||||
start_idx = chunk_id * self.block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
if start_idx < mask_num:
|
||||
start_chunk = max(0, cdiv(mask_num, self.block_size))
|
||||
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
|
||||
if chunk_mask is not None:
|
||||
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
|
||||
for chunk_id in range(start_chunk, max_chunks):
|
||||
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
|
||||
continue
|
||||
yield start_idx, end_idx, self._make_key_by_hash(h.hex())
|
||||
if chunk_id % put_step != put_step_rank:
|
||||
continue
|
||||
h = chunk_hashes[chunk_id]
|
||||
start_idx = chunk_id * self.block_size
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
yield start_idx, end_idx, h
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -473,6 +473,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._store_pressure_active = False
|
||||
self._skip_store_requests: set[str] = set()
|
||||
|
||||
# Per-request high-water mark of tokens actually persisted; the next
|
||||
# batch resumes here, so pressure-skipped or failed ranges are retried.
|
||||
self._saved_offset: dict[str, int] = {}
|
||||
|
||||
def add_stored_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
self.stored_requests[req_id] += 1
|
||||
@@ -487,6 +491,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if req_id in self.stored_requests:
|
||||
del self.stored_requests[req_id]
|
||||
self._skip_store_requests.discard(req_id)
|
||||
self._saved_offset.pop(req_id, None)
|
||||
|
||||
def _record_saved(self, req_id: str, token_len: int) -> None:
|
||||
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self._saved_offset[req_id] = token_len
|
||||
|
||||
def _should_skip_request(self, req_id: str) -> bool:
|
||||
with self.done_task_lock:
|
||||
@@ -526,6 +537,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
try:
|
||||
if token_len == 0:
|
||||
return
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -534,40 +546,42 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
return
|
||||
|
||||
# Resume from where this rank left off; only the new suffix is saved.
|
||||
save_start = self._saved_offset.get(req_id, 0)
|
||||
|
||||
# Within each lcm region only per-spec relevant chunks are loaded
|
||||
# (e.g., SWA or linear attn), so mask out irrelevant chunks
|
||||
store_masks = self.coord.store_mask(
|
||||
token_len, num_prompt_tokens=req_meta.num_prompt_tokens
|
||||
token_len,
|
||||
save_start,
|
||||
num_prompt_tokens=req_meta.num_prompt_tokens,
|
||||
)
|
||||
|
||||
starts: list[int] = []
|
||||
ends: list[int] = []
|
||||
keys: list[str] = []
|
||||
block_hashes: list[BlockHash] = []
|
||||
kv_event_block_hashes: list[BlockHash] = []
|
||||
group_indices: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = store_masks[g_idx]
|
||||
for chunk_idx, (start, end, key) in enumerate(
|
||||
db.process_tokens(token_len, req_meta.block_hashes)
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step_rank = (self.tp_rank + g_idx) % self.put_step
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len,
|
||||
req_meta.block_hashes,
|
||||
mask_num=save_start,
|
||||
chunk_mask=store_masks[g_idx],
|
||||
put_step=self.put_step,
|
||||
put_step_rank=put_step_rank,
|
||||
):
|
||||
if mask is not None and (
|
||||
chunk_idx >= len(mask) or not mask[chunk_idx]
|
||||
):
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
keys.append(key.to_string())
|
||||
block_hashes.append(BlockHash(bytes.fromhex(key.chunk_hash)))
|
||||
keys.append(db.key_for(block_hash))
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes.append(block_hash)
|
||||
group_indices.append(g_idx)
|
||||
|
||||
# Apply put_step striding for TP
|
||||
sl = slice(self.tp_rank % self.put_step, None, self.put_step)
|
||||
starts = starts[sl]
|
||||
ends = ends[sl]
|
||||
keys = keys[sl]
|
||||
block_hashes = block_hashes[sl]
|
||||
group_indices = group_indices[sl]
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_id, token_len)
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
@@ -593,13 +607,18 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
]
|
||||
|
||||
if not missing_indices:
|
||||
self._record_saved(req_id, token_len)
|
||||
return
|
||||
|
||||
starts = [starts[i] for i in missing_indices]
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
block_hashes = [block_hashes[i] for i in missing_indices]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
if len(missing_indices) != len(keys):
|
||||
starts = [starts[i] for i in missing_indices]
|
||||
ends = [ends[i] for i in missing_indices]
|
||||
keys = [keys[i] for i in missing_indices]
|
||||
if self.enable_kv_event:
|
||||
kv_event_block_hashes = [
|
||||
kv_event_block_hashes[i] for i in missing_indices
|
||||
]
|
||||
group_indices = [group_indices[i] for i in missing_indices]
|
||||
|
||||
logger.debug(
|
||||
"Storing KV cache for %d blocks (groups=%s) for request %s",
|
||||
@@ -612,8 +631,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
sizes: list[list[int]] = []
|
||||
stored_events: list[BlockStored] = []
|
||||
# parent_block_hash chains live within a group, not across.
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [maybe_convert_block_hash(bh) for bh in block_hashes]
|
||||
if self.enable_kv_event:
|
||||
prev_key_per_group: dict[int, Any] = {}
|
||||
new_block_hashes = [
|
||||
maybe_convert_block_hash(bh) for bh in kv_event_block_hashes
|
||||
]
|
||||
|
||||
for idx, (s, e, g_idx) in enumerate(
|
||||
zip(starts, ends, group_indices, strict=True)
|
||||
@@ -687,11 +709,13 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
"batch succeeds",
|
||||
req_id,
|
||||
)
|
||||
elif self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared after a "
|
||||
"successful store batch"
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_id, token_len)
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
"after a successful store batch"
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
@@ -775,7 +799,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
for start, end, key in db.process_tokens(
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len, req_meta.block_hashes, mask_num
|
||||
):
|
||||
chunk_idx = start // db.block_size
|
||||
@@ -784,7 +808,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
addr, size, block_id = db.prepare_value(
|
||||
start, end, req_meta.block_ids[g_idx]
|
||||
)
|
||||
key_list.append(key.to_string())
|
||||
key_list.append(db.key_for(block_hash))
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
block_id_list.append(block_id)
|
||||
|
||||
Reference in New Issue
Block a user