[Bugfix][Mooncake] Save the EAGLE attention proof with hybrid Mamba partial tails (#60695)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Dao007forever <daole@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Dao007forever
2026-10-10 00:08:08 +00:00
committed by GitHub
co-authored by Wei Zhao Claude Opus 5.5 zjy0516
parent 03a7d3776f
commit f8e6e365ae
11 changed files with 1142 additions and 390 deletions
@@ -3,7 +3,8 @@
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from collections.abc import Sequence
from typing import NamedTuple, cast
from dataclasses import replace
from typing import TYPE_CHECKING, NamedTuple, cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
@@ -13,12 +14,15 @@ from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
eagle_proof_margin,
resolve_dcp_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
@@ -26,6 +30,9 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
class StoreSpecGroup(NamedTuple):
spec: KVCacheSpec
@@ -82,7 +89,7 @@ class MooncakeStoreCoordinator:
hash_block_size: int,
use_eagle: bool = False,
retention_interval: int | None = None,
dcp_world_size: int = 1,
enable_partial_hash_hits: bool = False,
) -> None:
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
# to divide groups that participate in prefix caching. Non-shareable
@@ -110,14 +117,51 @@ class MooncakeStoreCoordinator:
}
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.enable_partial_hash_hits = partial_hash_hits_enabled(
kv_cache_groups, hash_block_size, dcp_world_size
)
# Core's decision: whether hits land on the hash unit, below the block.
self.enable_partial_hash_hits = enable_partial_hash_hits
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
self._verify_and_split_kv_cache_groups()
@classmethod
def from_kv_cache_config(
cls,
kv_cache_config: KVCacheConfig,
vllm_config: "VllmConfig",
scheduler_block_size: int,
hash_block_size: int,
) -> "MooncakeStoreCoordinator":
"""Build the coordinator the connector's scheduler and workers share.
Specs are DCP-resolved, so a block spans the tokens of one block-table
entry. Partial hits follow core's hit alignment, which checks every KV
cache group and its manager, not only the groups the store transfers;
unset (a config the engine core did not resolve) keeps hits on the
block.
"""
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
spec_config = vllm_config.speculative_config
alignment = kv_cache_config.cache_hit_alignment_tokens
return cls(
[
replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec, dcp_world_size
),
)
for group in kv_cache_config.prefix_cacheable_groups
],
scheduler_block_size,
hash_block_size,
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
retention_interval=kv_cache_config.prefix_cache_retention_interval,
enable_partial_hash_hits=(
alignment is not None and alignment < scheduler_block_size
),
)
def align_lookup_length(self, length: int) -> int:
alignment = (
self.hash_block_size
@@ -167,6 +211,17 @@ class MooncakeStoreCoordinator:
self.eagle_group_ids = {
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
}
self.eagle_proof_margin_by_group = {
gid: eagle_proof_margin(
group.spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and group.manager_cls.supports_fine_grained_hash_lookup,
)
for group in attention_groups
if group.use_eagle and not isinstance(group.spec, MambaSpec)
for gid in group.group_ids
}
def find_longest_cache_hit(
self,
@@ -394,14 +449,16 @@ class MooncakeStoreCoordinator:
# never drops a block, so a widened bound would match past the
# attention-verified hit and resume from speculative state (#43559).
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = (
self.hash_block_size
if self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup
and spec.block_size > self.hash_block_size
else spec.block_size
eagle_margin = eagle_proof_margin(
spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup,
)
_max_length = min(
curr_hit_length + eagle_margin,
len(block_hashes) * self.hash_block_size,
)
_max_length = min(curr_hit_length + eagle_margin, max_length)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
@@ -449,20 +506,3 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
def partial_hash_hits_enabled(
kv_cache_groups: Sequence[KVCacheGroupSpec],
hash_block_size: int,
dcp_world_size: int = 1,
) -> bool:
"""Match core's DCP-aware Mamba partial-hit condition."""
return any(
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
and spec.mamba_cache_mode == "align"
and (
(dcp_world_size == 1 and spec.block_size > hash_block_size)
or (dcp_world_size > 1 and spec.block_size >= hash_block_size)
)
for g in kv_cache_groups
)
@@ -7,7 +7,7 @@
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from typing import cast
from typing import NamedTuple, cast
import numpy as np
import torch
@@ -712,6 +712,7 @@ class RequestTracker:
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
prefill_end_tokens: int = 0
partial_tail_sent: bool = False
def reset(self) -> None:
self.token_len = 0
@@ -720,6 +721,7 @@ class RequestTracker:
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
self.partial_tail_sent = False
def update(
self,
@@ -739,6 +741,14 @@ class RequestTracker:
existing.extend(new)
class BoundaryPut(NamedTuple):
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
group_id: int
block_id: int
num_tokens: int
@dataclass
class ReqMeta:
"""Per-request metadata for store put/get operations."""
@@ -756,15 +766,24 @@ class ReqMeta:
# Absolute request offset represented by token_ids[0].
token_ids_start: int = 0
num_prompt_tokens: int | None = None
# Total prefill tokens, including generated tokens replayed after preemption.
prefill_end_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Core-provided (group_id, block_id, boundary_tokens) mamba "align"
# boundary states. A block-aligned entry is a committed boundary snapshot;
# a non-aligned entry is the sub-block CoW tail. The store-job reference
# keeps each exact block alive until every worker rank finishes the job.
boundary_state_offloads: list[tuple[int, int, int]] | None = None
# Blocks stored under an explicit prefix hash rather than by position:
# core-provided mamba "align" states (a block-aligned entry is a committed
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
# store-job reference keeps each block alive until every worker rank
# finishes the job.
boundary_puts: list[BoundaryPut] | None = None
# Total computed prefix length at the end of this step.
completed_token_len: int | None = None
# Set on the save that first covers the whole prompt, so the scheduler
# publishes the prompt's partial tail once rather than on every save.
publish_partial_tail: bool = False
@staticmethod
def from_request_tracker(
@@ -773,8 +792,12 @@ class ReqMeta:
load_spec: LoadSpec | None = None,
skip_save: bool | None = False,
block_hashes: list[BlockHash] | None = None,
save_partial_tail: bool = False,
num_prompt_tokens: int | None = None,
) -> "ReqMeta | None":
"""Create ReqMeta from a RequestTracker."""
if save_partial_tail:
assert num_prompt_tokens is not None
if block_hashes is None:
block_hashes = []
input_token_len = tracker.token_len
@@ -783,7 +806,15 @@ class ReqMeta:
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
num_tokens_to_save = input_token_len // block_size * block_size
skip_save = skip_save or num_tokens_to_save < chunk_boundary
publish_tail = (
save_partial_tail
and not tracker.partial_tail_sent
and num_prompt_tokens is not None
and 0 < num_prompt_tokens <= input_token_len
)
skip_save = skip_save or (
num_tokens_to_save < chunk_boundary and not publish_tail
)
# A ReqMeta must never carry both a save AND a load.
# The save would also be wasted work — the bytes are being looked up
# in the store right now. Later cached_reqs steps save new tokens
@@ -795,6 +826,7 @@ class ReqMeta:
if not skip_save:
tracker.num_saved_tokens = num_tokens_to_save
tracker.partial_tail_sent |= publish_tail
token_ids = None
if tracker.token_ids and not skip_save:
@@ -826,7 +858,10 @@ class ReqMeta:
block_hashes=block_hashes,
token_ids=token_ids,
token_ids_start=token_ids_start,
num_prompt_tokens=tracker.prefill_end_tokens,
num_prompt_tokens=num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
completed_token_len=input_token_len,
publish_partial_tail=publish_tail and not skip_save,
)
@@ -5,14 +5,17 @@
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from collections.abc import Sequence
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
partial_hash_hits_enabled,
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
BoundaryPut,
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
@@ -23,12 +26,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.kv_cache_utils import (
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
MambaSpec,
get_mamba_prefill_checkpoint_position,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
@@ -49,6 +59,79 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
return request.prompt_token_ids
def _partial_tail_non_mamba_puts(
coord: MooncakeStoreCoordinator,
req_meta: ReqMeta,
block_sizes: Sequence[int],
) -> list[BoundaryPut]:
"""Locate the non-Mamba blocks a partial-tail save publishes.
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
if both pieces of KV are stored:
- Mamba groups: the state at ``boundary``, handed off by the core as a
boundary put of its own, so Mamba groups are not listed here.
- Other groups: the KV from the last LCM-aligned normal save up to
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
block is keyed by the hash at its end, clipped to ``proof_end``.
Groups whose proof is not computed yet are omitted. For example, with LCM
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
are 8-10, keyed at 36, 40 and 44.
A Mamba state handed off at any other position, i.e. a shared-prefix
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
``block_sizes`` are the per-group block sizes, indexed like
``req_meta.block_ids``.
"""
mamba_tails = [
put.num_tokens
for put in req_meta.boundary_puts or []
if put.group_id in coord.mamba_group_ids
and put.num_tokens % block_sizes[put.group_id]
]
# The tail is due on the save that first covers the prompt, or when Mamba
# hands off its checkpoint state.
if not mamba_tails and not req_meta.publish_partial_tail:
return []
prompt_tokens = req_meta.num_prompt_tokens
completed = req_meta.completed_token_len
assert prompt_tokens is not None and completed is not None
hash_block_size = coord.hash_block_size
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
# groups carry the eagle flag.
boundary = get_mamba_prefill_checkpoint_position(
prompt_tokens, hash_block_size, coord.use_eagle
)
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
# attention KV, EAGLE proof included, ends on an attention block end no later
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
return []
num_hashes = len(req_meta.block_hashes)
if boundary == 0 or boundary // hash_block_size > num_hashes:
return []
start = boundary // coord.lcm_block_size * coord.lcm_block_size
puts: list[BoundaryPut] = []
for group_id, block_size in enumerate(block_sizes):
if group_id in coord.mamba_group_ids:
continue
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
if proof_end > completed or proof_end // hash_block_size > num_hashes:
continue
group_blocks = req_meta.block_ids[group_id]
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
if block_idx >= len(group_blocks):
break
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
continue
num_tokens = min((block_idx + 1) * block_size, proof_end)
puts.append(BoundaryPut(group_id, block_id, num_tokens))
return puts
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
@@ -84,11 +167,6 @@ class MooncakeStoreScheduler:
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.enable_partial_hash_hits = partial_hash_hits_enabled(
store_groups,
self._hash_block_size,
vllm_config.parallel_config.decode_context_parallel_size,
)
mamba_groups = {
group_id: group.kv_cache_spec
for group_id, group in enumerate(store_groups)
@@ -99,6 +177,11 @@ class MooncakeStoreScheduler:
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._boundary_state_group_ids = frozenset(mamba_groups)
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
@@ -273,6 +356,8 @@ class MooncakeStoreScheduler:
# producer. Loads are still carried by the same metadata.
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
@@ -323,6 +408,8 @@ class MooncakeStoreScheduler:
load_spec=load_spec,
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
else:
# Decode/chunked request
@@ -364,6 +451,8 @@ class MooncakeStoreScheduler:
load_spec=None,
skip_save=False,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
@@ -398,6 +487,7 @@ class MooncakeStoreScheduler:
load_spec=load_spec,
skip_save=None,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
)
if req_meta is not None:
meta.add_request(req_meta)
@@ -455,6 +545,8 @@ class MooncakeStoreScheduler:
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
coord = self._store_coord
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
for req_meta in meta.requests:
if not req_meta.can_save:
continue
@@ -464,27 +556,27 @@ class MooncakeStoreScheduler:
if req_meta.store_job_id is not None:
assert req_meta.store_job_id in self._pinned_saves
continue
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids: list[int] = []
if req_meta.boundary_state_offloads:
block_ids.extend(
block_id for _, block_id, _ in req_meta.boundary_state_offloads
)
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
assert NULL_BLOCK_ID not in block_ids, (
"A null block cannot back a boundary-state offload"
)
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
"A null block cannot back a boundary put"
)
if req_meta.token_len_chunk:
# Normal prefix save: pin all non-Mamba sources for retries.
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
)
# An aligned boundary block may also be present in the request's
# block table. Take and release exactly one reference per block.
block_ids = list(dict.fromkeys(block_ids))
@@ -515,17 +607,39 @@ class MooncakeStoreScheduler:
if boundary_tokens > tracker.prefill_end_tokens:
return False
pinned_block_ids: list[int] = []
remapped_offloads: list[tuple[int, int, int]] = []
mamba_puts: list[BoundaryPut] = []
for group_id, block_id, boundary in partial_tail_offloads:
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
return False
if block_id == NULL_BLOCK_ID:
return False
pinned_block_ids.append(block_id)
remapped_offloads.append((store_group_id, block_id, boundary))
pinned_block_ids = list(dict.fromkeys(pinned_block_ids))
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
req_meta = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(
block_ids[group_id].copy() for group_id in self._store_group_ids
),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=request.num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=mamba_puts,
completed_token_len=request.num_computed_tokens,
)
coord = self._store_coord
req_meta.boundary_puts = [
*_partial_tail_non_mamba_puts(
coord,
req_meta,
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
),
*mamba_puts,
]
pinned_block_ids = list(
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
)
pool = self._gpu_block_pool
assert pool is not None, (
@@ -537,18 +651,8 @@ class MooncakeStoreScheduler:
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
self._finished_partial_tail_metas[request.request_id] = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(
block_ids[group_id].copy() for group_id in self._store_group_ids
),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
store_job_id=store_job_id,
boundary_state_offloads=remapped_offloads,
)
req_meta.store_job_id = store_job_id
self._finished_partial_tail_metas[request.request_id] = req_meta
tracker.has_pending_offload = True
# The store job owns exact block refs, so request cleanup need not wait.
return False
@@ -575,7 +679,7 @@ class MooncakeStoreScheduler:
# going away, so the offload is conservatively dropped.
logger.debug("Dropping boundary-state offload for request %s", req_id)
continue
accepted: list[tuple[int, int, int]] = []
accepted: list[BoundaryPut] = []
for group_id, block_id, boundary_tokens in entries:
# Every other group stops saving at the end of this prefill, so
# a mamba-only key past it can never complete a joint hybrid
@@ -589,12 +693,12 @@ class MooncakeStoreScheduler:
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
continue
accepted.append((store_group_id, block_id, boundary_tokens))
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
if not accepted:
continue
tracker.has_pending_offload = True
if (req_meta := save_metas.get(req_id)) is not None:
req_meta.boundary_state_offloads = accepted
req_meta.boundary_puts = accepted
continue
meta.add_request(
ReqMeta(
@@ -603,8 +707,10 @@ class MooncakeStoreScheduler:
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
boundary_state_offloads=accepted,
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=accepted,
completed_token_len=tracker.token_len,
)
)
@@ -74,7 +74,6 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
maybe_convert_block_hash,
resolve_dcp_kv_cache_spec,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
@@ -569,40 +568,43 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.clear()
return True
def _boundary_snapshot_puts(
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
def _boundary_puts(
self, req_meta: ReqMeta
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
"""Puts for committed mamba "align" boundary-state snapshots.
"""Puts for ``req_meta.boundary_puts``: each block under the hash of
the prefix ending at its ``num_tokens``.
These are block-aligned boundaries, i.e. exactly what the normal save
would key — but ``store_mask`` masks mamba groups out of it entirely, so
this is their *only* writer. The exclusion is not an optimization: the
normal save resolves a chunk's address as
``req_meta.block_ids[g][start // block_size]``, and ``block_ids`` is the
connector's append-only mirror of the core's per-group table. An
align-mode table is mutated in place (a superseded state block is freed
and nulled; speculative blocks relocate), and the connector is never
told, so a stale mirror entry is indistinguishable from a live one — a
retry of a failed or pressure-skipped chunk would read a block that now
belongs to another request.
Each entry's handed-off block *is* the boundary state and is pinned by
the core, so it is uploaded under its boundary-end hash key and never
resolved positionally.
The scheduler resolves and pins every block, so none is looked up
positionally here. That matters for mamba "align" states: ``store_mask``
masks mamba groups out of the normal save, which resolves a chunk's
address as ``req_meta.block_ids[g][start // block_size]``. An align-mode
table is mutated in place (a superseded state block is freed and
nulled; speculative blocks relocate) without telling the connector, so
a retried positional read could hit a block that now belongs to another
request. The handed-off block *is* the state.
"""
hash_block_size = self.coord.hash_block_size
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
for group_id, block_id, boundary in entries:
if boundary == 0 or block_id == NULL_BLOCK_ID:
for group_id, block_id, num_tokens in req_meta.boundary_puts or []:
if (
not self.group_participates[group_id]
or num_tokens == 0
or block_id == NULL_BLOCK_ID
):
continue
hash_idx = boundary // hash_block_size - 1
# A negative index would silently key the block by the last hash.
assert num_tokens % hash_block_size == 0, (
f"Boundary put at {num_tokens} tokens is not a multiple of the "
f"hash block size {hash_block_size}"
)
hash_idx = num_tokens // hash_block_size - 1
if hash_idx >= len(req_meta.block_hashes):
continue
db = self.token_databases[group_id]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[group_id]
put_step_rank = (self.tp_rank + group_id) % put_step
if (boundary // db.block_size - 1) % put_step != put_step_rank:
if (cdiv(num_tokens, db.block_size) - 1) % put_step != put_step_rank:
continue
addr, size = db.prepare_value_for_block(block_id)
puts.append(
@@ -610,115 +612,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
return puts
def _sub_block_tail_puts(
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
"""Puts for the request's sub-block partial tail (its last prompt hash
boundary), so a later request can hit the sub-block prefix.
Covers every group's blocks from the normal save's lcm floor to the
boundary: the normal save floors to ``lcm_block_size``, so a
smaller-block group's full blocks in that gap are never persisted
elsewhere, and the consumer's lookup needs every group at every probed
boundary. Full blocks are keyed by their block-end hash and the partial
boundary block by the boundary sub-hash; a mamba "align" group
contributes only its boundary block, from the core-provided CoW block.
"""
boundaries = {boundary for _, _, boundary in entries}
if len(boundaries) != 1:
raise ValueError(
"Sub-block partial-tail offloads for one request must share a boundary"
)
boundary = boundaries.pop()
hash_block_size = self.coord.hash_block_size
if boundary == 0 or boundary // hash_block_size - 1 >= len(
req_meta.block_hashes
):
return []
mamba_offloads = {group_id: block_id for group_id, block_id, _ in entries}
saved = self._saved_offset.get(req_meta.req_id, 0)
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
for g_idx, db in enumerate(self.token_databases):
if not self.group_participates[g_idx]:
continue
group_blocks = req_meta.block_ids[g_idx]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[g_idx]
put_step_rank = (self.tp_rank + g_idx) % put_step
# Always include the boundary block: its sub-hash key is written
# only here, even if normal saves already advanced past it.
last_block = cdiv(boundary, db.block_size) - 1
for block_idx in range(
min(saved // db.block_size, last_block), last_block + 1
):
if block_idx % put_step != put_step_rank:
continue
valid_end = min((block_idx + 1) * db.block_size, boundary)
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
if g_idx in mamba_offloads:
if valid_end != boundary:
# Interior align-mode state positions are null or
# stale (the block table is not append-only) and never
# valid gap content; only the boundary block is
# persisted, from the core-provided hand-off.
continue
block_id = mamba_offloads[g_idx]
elif g_idx in self.coord.mamba_group_ids:
continue
elif block_idx < len(group_blocks):
block_id = group_blocks[block_idx]
else:
continue
if block_id == NULL_BLOCK_ID:
logger.debug(
"Skipping unavailable partial-tail source block "
"(req=%s, group=%d, block=%d)",
req_meta.req_id,
g_idx,
block_idx,
)
continue
addr, size = db.prepare_value_for_block(block_id)
puts.append((db.key_for(key_hash), addr, size, db.metadata))
return puts
def _maybe_offload_boundary_states(self, req_meta: ReqMeta) -> bool:
"""Persist connector-pinned mamba "align" boundary states handed off
for this request, deduped against the store.
"""Persist this job's boundary puts, deduped against the store.
This is every mamba key the connector writes — ``store_mask`` excludes
mamba groups from the positional normal save, aligned boundaries
included (see :meth:`_boundary_snapshot_puts`).
The two entry kinds are keyed and sourced differently, so they are
prepared separately and put in one batch:
- block-aligned for its group: a committed boundary-state snapshot,
the handed-off block itself;
- not block-aligned: the sub-block CoW partial tail, which also has to
cover the other groups' blocks in the normal save's lcm gap.
These are every mamba key the connector writes (``store_mask`` excludes
mamba groups from the positional normal save) plus the other groups'
partial-tail blocks in the normal save's lcm gap.
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
offloads = req_meta.boundary_state_offloads
if not offloads or not req_meta.block_hashes:
if not req_meta.boundary_puts or not req_meta.block_hashes:
return True
snapshots: list[tuple[int, int, int]] = []
sub_block: list[tuple[int, int, int]] = []
for group_id, block_id, boundary in offloads:
entry = (group_id, block_id, boundary)
if boundary % self.token_databases[group_id].block_size == 0:
snapshots.append(entry)
else:
sub_block.append(entry)
puts = self._boundary_snapshot_puts(req_meta, snapshots)
if sub_block and self.coord.enable_partial_hash_hits:
puts.extend(self._sub_block_tail_puts(req_meta, sub_block))
puts = self._boundary_puts(req_meta)
if not puts:
return True
@@ -854,9 +761,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
# Offload the handed-off mamba boundary states (independent of the
# normal positional save, which may be skipped this step).
if req_meta.boundary_state_offloads is not None and not (
self._maybe_offload_boundary_states(req_meta)
):
if not self._maybe_offload_boundary_states(req_meta):
return
if token_len == 0:
@@ -870,7 +775,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
store_masks = self.coord.store_mask(
token_len,
save_start,
num_prompt_tokens=req_meta.num_prompt_tokens,
num_prompt_tokens=req_meta.prefill_end_tokens,
)
starts: list[int] = []
@@ -1603,31 +1508,10 @@ class MooncakeStoreWorker:
)
return
self._kv_cache_groups = [
dataclasses.replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec,
self.dcp_size,
),
)
for group in kv_cache_config.prefix_cacheable_groups
]
spec_cfg = getattr(vllm_config, "speculative_config", None)
use_eagle_block_drop = bool(
spec_cfg.use_eagle_block_drop()
if spec_cfg is not None
and callable(getattr(spec_cfg, "use_eagle_block_drop", None))
else False
)
self.coord = MooncakeStoreCoordinator(
self._kv_cache_groups,
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
use_eagle=use_eagle_block_drop,
retention_interval=kv_cache_config.prefix_cache_retention_interval,
dcp_world_size=self.dcp_size,
self.coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self.block_size, self.hash_block_size
)
self._kv_cache_groups = self.coord.kv_cache_groups
self.store_tp_size, store_namespace, store_layout_cls = (
self._select_store_layout(extra_config)
)