[3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Dao007forever
2026-07-26 23:22:17 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 544cb724c8
commit d742856610
13 changed files with 1448 additions and 49 deletions
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
@@ -83,6 +84,9 @@ class MooncakeStoreCoordinator:
self.kv_cache_groups = kv_cache_groups
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_groups, hash_block_size
)
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
@@ -94,7 +98,12 @@ class MooncakeStoreCoordinator:
self._verify_and_split_kv_cache_groups()
def align_lookup_length(self, length: int) -> int:
return length // self.lcm_block_size * self.lcm_block_size
alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
return length // alignment * alignment
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
@@ -226,9 +235,14 @@ class MooncakeStoreCoordinator:
retention_interval: int | None,
num_prompt_tokens: int | None,
) -> tuple[list[bool] | None, ...]:
assert aligned_token_len % self.lcm_block_size == 0, (
mask_alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
assert aligned_token_len % mask_alignment == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"lcm_block_size ({self.lcm_block_size})"
f"{mask_alignment}"
)
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
@@ -278,17 +292,21 @@ class MooncakeStoreCoordinator:
one already removed by the lookup.
"""
eagle_indices = self.eagle_attn_group_indices if apply_eagle else set()
alignment_tokens = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls = self.attention_groups[0]
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=(0 in eagle_indices),
alignment_tokens=spec.block_size,
alignment_tokens=alignment_tokens,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
@@ -320,17 +338,23 @@ class MooncakeStoreCoordinator:
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
_max_length = curr_hit_length
if drop_eagle_block:
_max_length = min(curr_hit_length + spec.block_size, max_length)
hashes = self.block_hashes_for_spec(block_hashes, spec)
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = (
self.hash_block_size
if self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup
and spec.block_size > self.hash_block_size
else spec.block_size
)
_max_length = min(curr_hit_length + eagle_margin, max_length)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=self.lcm_block_size,
alignment_tokens=alignment_tokens,
)
if drop_eagle_block:
eagle_verified.add(idx)
@@ -348,10 +372,11 @@ class MooncakeStoreCoordinator:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic.
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
spec0, group_ids0, _ = self.attention_groups[0]
if isinstance(spec0, FullAttentionSpec):
num_blocks = hit_length // spec0.block_size
num_blocks = cdiv(hit_length, spec0.block_size)
for gid in group_ids0:
full_blks = hit_blocks_by_group[gid]
assert full_blks is not None
@@ -368,3 +393,18 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
def partial_hash_hits_enabled(
kv_cache_groups: list[KVCacheGroupSpec], hash_block_size: int
) -> bool:
"""Mirror of core's ``HybridKVCacheCoordinator.enable_partial_hash_hits``
(its dcp == 1 clause holds: the connector rejects hybrid + DCP/PCP > 1).
Single copy on purpose — scheduler and coordinator must not disagree.
"""
return any(
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
and spec.mamba_cache_mode == "align"
and spec.block_size > hash_block_size
for g in kv_cache_groups
)
@@ -228,16 +228,28 @@ class ChunkedTokenDatabase:
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert not (spans % self.block_size).any()
assert not (spans % self.hash_block_size).any()
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
sizes = blen[None, :] * (spans // self.block_size)[:, None]
block_counts = (spans + self.block_size - 1) // self.block_size
sizes = blen[None, :] * block_counts[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
"""Return addresses and sizes for one physical block slot."""
addr_list = []
size_list = []
length = len(self.block_len)
for index, base_addr in enumerate(self.kv_caches_base_addr):
addr = base_addr + block_id * self.block_len[index % length]
addr_list.append(addr)
size_list.append(self.block_len[index % length])
return addr_list, size_list
def process_tokens(
self,
token_len: int,
@@ -256,7 +268,8 @@ class ChunkedTokenDatabase:
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens.
token_len: Total number of tokens. Must be hash-block aligned and
covered by ``block_hashes`` when hashes are present.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
@@ -269,11 +282,10 @@ class ChunkedTokenDatabase:
assert put_step > 0
if not block_hashes:
return
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, self.block_size
)
assert token_len % self.hash_block_size == 0
assert token_len // self.hash_block_size <= len(block_hashes)
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
max_chunks = cdiv(token_len, self.block_size)
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
@@ -281,9 +293,9 @@ class ChunkedTokenDatabase:
continue
if chunk_id % put_step != put_step_rank:
continue
h = chunk_hashes[chunk_id]
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
h = block_hashes[end_idx // self.hash_block_size - 1]
yield start_idx, end_idx, h
@@ -306,6 +318,7 @@ class RequestTracker:
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
has_pending_offload: bool = False
# Snapshot of the prefill range length at tracker creation time.
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
@@ -316,6 +329,7 @@ class RequestTracker:
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
def update(
@@ -352,6 +366,11 @@ class ReqMeta:
token_ids: list[int] | None = None
num_prompt_tokens: int | None = None
# Core-provided per-mamba-group
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
# Present only on the producer's CoW step; drives the connector's offload
# (the FA group's block is derived from block_ids and boundary_tokens).
partial_tail_offloads: list[tuple[int, int, int]] | None = None
@staticmethod
def from_request_tracker(
@@ -11,6 +11,9 @@ from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
partial_hash_hits_enabled,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
LoadSpec,
MooncakeStoreConnectorMetadata,
@@ -63,6 +66,9 @@ class MooncakeStoreScheduler:
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_config.kv_cache_groups, self._hash_block_size
)
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
@@ -80,7 +86,12 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if request.num_tokens < self._block_size:
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
)
if request.num_tokens < align:
return 0, False
num_external_hit_tokens = self.client.lookup(
@@ -338,6 +349,44 @@ class MooncakeStoreScheduler:
if req_meta is not None:
meta.add_request(req_meta)
# Flush partial-tail offloads in the step they arrive: the CoW copy is
# enqueued before the connector event records, so this step's event
# fences the cow block. Ride the request's save meta when present, else
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
# save; can_save=True takes the normal enqueue path).
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
if step_partial_tails and not force_skip_save:
pending = dict(step_partial_tails)
for req_meta in meta.requests:
if req_meta.can_save:
groups = pending.pop(req_meta.req_id, None)
if groups:
req_meta.partial_tail_offloads = groups
tracker = self._request_trackers.get(req_meta.req_id)
if tracker is not None:
tracker.has_pending_offload = True
for req_id, groups in pending.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks
# are going away, so the offload is conservatively dropped.
logger.debug("Dropping partial-tail offload for request %s", req_id)
continue
assert len({boundary for _, _, boundary in groups}) == 1
tracker.has_pending_offload = True
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
partial_tail_offloads=groups,
)
)
return meta
def request_finished(
@@ -352,7 +401,9 @@ class MooncakeStoreScheduler:
# Missing tracker can happen when the request is aborted before the
# connector observes the normal finished lifecycle or is preempted
# before finishing.
if tracker is None or tracker.num_saved_tokens <= 0:
if tracker is None or (
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
):
return False, None
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
@@ -61,6 +61,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
@@ -531,6 +532,164 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.clear()
return True
def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool:
"""Offload the request's sub-block partial tail (its last prompt hash
boundary) so a later request can hit the sub-block prefix.
Covers every block from the normal save's lcm floor to the boundary:
the normal save floors to ``lcm_block_size``, so a smaller-block
group's full blocks in that gap are never persisted elsewhere, and
the consumer's lookup needs every group at every probed boundary.
Full blocks are keyed by their block-end hash, the partial boundary
block by the boundary sub-hash; the mamba "align" boundary block is
the core-provided CoW block. All keys are deduped against the store.
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes:
return True
partial_tail_offloads = req_meta.partial_tail_offloads
if not partial_tail_offloads:
return True
hash_block_size = self.coord.hash_block_size
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary = boundaries.pop()
if boundary == 0:
return True
if boundary // hash_block_size - 1 >= len(req_meta.block_hashes):
return True
mamba_offloads = {
group_id: block_id for group_id, block_id, _ in partial_tail_offloads
}
keys: list[str] = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
saved = self._saved_offset.get(req_meta.req_id, 0)
for g_idx, db in enumerate(self.token_databases):
group_blocks = req_meta.block_ids[g_idx]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[g_idx]
put_step_rank = (self.tp_rank + g_idx) % put_step
# Always include the boundary block: its sub-hash key is written
# only here, even if normal saves already advanced past it.
last_block = cdiv(boundary, db.block_size) - 1
for block_idx in range(
min(saved // db.block_size, last_block), last_block + 1
):
if block_idx % put_step != put_step_rank:
continue
valid_end = min((block_idx + 1) * db.block_size, boundary)
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
if (
g_idx in mamba_offloads
and valid_end == boundary
and boundary % db.block_size != 0
):
block_id = mamba_offloads[g_idx]
else:
if block_idx >= len(group_blocks):
continue
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
logger.debug(
"Skipping unavailable partial-tail source block "
"(req=%s, group=%d, block=%d)",
req_meta.req_id,
g_idx,
block_idx,
)
continue
addr, size = db.prepare_value_for_block(block_id)
keys.append(db.key_for(key_hash))
addrs.append(addr)
sizes.append(size)
if not keys:
return True
exists_start = time.perf_counter()
try:
exists = self.store.batch_is_exist(keys)
except Exception as e:
self._record_operation(
"save_exists",
exists_start,
len(keys),
status="error",
num_failed_keys=len(keys),
)
logger.error(
"Failed to check partial-tail keys for request %s: %s",
req_meta.req_id,
e,
)
return False
self._record_operation("save_exists", exists_start, len(keys))
missing = [i for i, e in enumerate(exists) if e != 1]
if not missing:
return True
keys = [keys[i] for i in missing]
addrs = [addrs[i] for i in missing]
sizes = [sizes[i] for i in missing]
if req_meta.current_event is not None:
# Fence the CoW block copy enqueued earlier this step.
req_meta.current_event.synchronize()
batch_bytes = _sum_batch_bytes(sizes)
put_start = time.perf_counter()
try:
res = self.store.batch_put_from_multi_buffers(
keys, addrs, sizes, self.replicate_config
)
except Exception as e:
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="error",
num_failed_keys=len(keys),
)
logger.error(
"Failed to put partial-tail keys for request %s: %s",
req_meta.req_id,
e,
)
return False
failed = [i for i, value in enumerate(res) if value < 0]
self._record_operation(
"save_put",
put_start,
len(keys),
num_bytes=batch_bytes,
status="partial_failure" if failed else "ok",
num_failed_keys=len(failed),
)
if failed:
failed_codes = {res[i] for i in failed}
logger.warning(
"Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)",
req_meta.req_id,
len(failed),
len(keys),
failed_codes,
)
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
self._mark_request_skipped_for_pressure(req_meta.req_id)
return False
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared after a "
"successful partial-tail batch"
)
return True
def _handle_request(self, req_meta: ReqMeta):
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
# is also ``store_mask``'s precondition.
@@ -548,9 +707,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
# so the scheduler can release the GPU blocks it pinned for this
# request (via `delay_free_blocks`) even when the store path raises.
try:
if token_len == 0:
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -559,6 +715,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
return
# Offload the sub-block partial tail (independent of the normal
# block-aligned save, which may be skipped this step).
if req_meta.partial_tail_offloads is not None and not (
self._maybe_offload_partial_tail(req_meta)
):
return
if token_len == 0:
return
# Resume from where this rank left off; only the new suffix is saved.
save_start = self._saved_offset.get(req_id, 0)
@@ -1385,7 +1551,7 @@ class MooncakeStoreWorker:
self.recv_request_queue.put(request)
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization
# Issue stores with CUDA event synchronization.
if self.kv_role in ["kv_producer", "kv_both"]:
current_event = None
for request in meta.requests:
@@ -1501,21 +1667,32 @@ class MooncakeStoreWorker:
# candidate_meta stores the (group, hash_bytes) for key slice.
candidate_keys: list[str] = []
candidate_meta: list[tuple[int, bytes]] = []
lookup_masks = self.coord.lookup_mask(token_len)
fine_grained = self.coord.enable_partial_hash_hits
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
for g_idx, db in enumerate(self.token_dbs):
spec_block_size = db.block_size
lookup_mask = lookup_masks[g_idx]
key_prefixes = self._lookup_key_prefixes[g_idx]
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
mask_limit = (
max_chunks if lookup_mask is None else min(max_chunks, len(lookup_mask))
)
for chunk_id in range(mask_limit):
if lookup_mask is not None and not lookup_mask[chunk_id]:
continue
if fine_grained:
max_units = min(len(block_hashes), token_len // self.hash_block_size)
unit_ids: range | list[int] = range(max_units)
group_hashes: Sequence[BlockHash] = block_hashes
else:
lookup_mask = lookup_masks[g_idx] # type: ignore[index]
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
mask_limit = (
max_chunks
if lookup_mask is None
else min(max_chunks, len(lookup_mask))
)
unit_ids = [
chunk_id
for chunk_id in range(mask_limit)
if lookup_mask is None or lookup_mask[chunk_id]
]
for chunk_id in unit_ids:
h = group_hashes[chunk_id]
hash_hex = h.hex()
for key_prefix in key_prefixes: