[Bugfix][Mooncake] Save the EAGLE attention proof with hybrid Mamba partial tails (#60695)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Signed-off-by: Dao007forever <daole@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Wei Zhao
Claude Opus 5.5
zjy0516
parent
03a7d3776f
commit
f8e6e365ae
@@ -3,7 +3,8 @@
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import NamedTuple, cast
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, NamedTuple, cast
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
@@ -13,12 +14,15 @@ from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
eagle_proof_margin,
|
||||
resolve_dcp_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
@@ -26,6 +30,9 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class StoreSpecGroup(NamedTuple):
|
||||
spec: KVCacheSpec
|
||||
@@ -82,7 +89,7 @@ class MooncakeStoreCoordinator:
|
||||
hash_block_size: int,
|
||||
use_eagle: bool = False,
|
||||
retention_interval: int | None = None,
|
||||
dcp_world_size: int = 1,
|
||||
enable_partial_hash_hits: bool = False,
|
||||
) -> None:
|
||||
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
|
||||
# to divide groups that participate in prefix caching. Non-shareable
|
||||
@@ -110,14 +117,51 @@ class MooncakeStoreCoordinator:
|
||||
}
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
kv_cache_groups, hash_block_size, dcp_world_size
|
||||
)
|
||||
# Core's decision: whether hits land on the hash unit, below the block.
|
||||
self.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
self.use_eagle = use_eagle
|
||||
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
|
||||
self.retention_interval = retention_interval
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
@classmethod
|
||||
def from_kv_cache_config(
|
||||
cls,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> "MooncakeStoreCoordinator":
|
||||
"""Build the coordinator the connector's scheduler and workers share.
|
||||
|
||||
Specs are DCP-resolved, so a block spans the tokens of one block-table
|
||||
entry. Partial hits follow core's hit alignment, which checks every KV
|
||||
cache group and its manager, not only the groups the store transfers;
|
||||
unset (a config the engine core did not resolve) keeps hits on the
|
||||
block.
|
||||
"""
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
spec_config = vllm_config.speculative_config
|
||||
alignment = kv_cache_config.cache_hit_alignment_tokens
|
||||
return cls(
|
||||
[
|
||||
replace(
|
||||
group,
|
||||
kv_cache_spec=resolve_dcp_kv_cache_spec(
|
||||
group.kv_cache_spec, dcp_world_size
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.prefix_cacheable_groups
|
||||
],
|
||||
scheduler_block_size,
|
||||
hash_block_size,
|
||||
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
enable_partial_hash_hits=(
|
||||
alignment is not None and alignment < scheduler_block_size
|
||||
),
|
||||
)
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
alignment = (
|
||||
self.hash_block_size
|
||||
@@ -167,6 +211,17 @@ class MooncakeStoreCoordinator:
|
||||
self.eagle_group_ids = {
|
||||
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
|
||||
}
|
||||
self.eagle_proof_margin_by_group = {
|
||||
gid: eagle_proof_margin(
|
||||
group.spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and group.manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
for group in attention_groups
|
||||
if group.use_eagle and not isinstance(group.spec, MambaSpec)
|
||||
for gid in group.group_ids
|
||||
}
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
@@ -394,14 +449,16 @@ class MooncakeStoreCoordinator:
|
||||
# never drops a block, so a widened bound would match past the
|
||||
# attention-verified hit and resume from speculative state (#43559).
|
||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||
eagle_margin = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
and manager_cls.supports_fine_grained_hash_lookup
|
||||
and spec.block_size > self.hash_block_size
|
||||
else spec.block_size
|
||||
eagle_margin = eagle_proof_margin(
|
||||
spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
_max_length = min(
|
||||
curr_hit_length + eagle_margin,
|
||||
len(block_hashes) * self.hash_block_size,
|
||||
)
|
||||
_max_length = min(curr_hit_length + eagle_margin, max_length)
|
||||
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=_max_length,
|
||||
@@ -449,20 +506,3 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
|
||||
|
||||
def partial_hash_hits_enabled(
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec],
|
||||
hash_block_size: int,
|
||||
dcp_world_size: int = 1,
|
||||
) -> bool:
|
||||
"""Match core's DCP-aware Mamba partial-hit condition."""
|
||||
return any(
|
||||
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
|
||||
and spec.mamba_cache_mode == "align"
|
||||
and (
|
||||
(dcp_world_size == 1 and spec.block_size > hash_block_size)
|
||||
or (dcp_world_size > 1 and spec.block_size >= hash_block_size)
|
||||
)
|
||||
for g in kv_cache_groups
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -712,6 +712,7 @@ class RequestTracker:
|
||||
# For a fresh request this is len(prompt). For a resumed-from-preemption
|
||||
# request it includes previously-generated tokens, which are re-prefilled.
|
||||
prefill_end_tokens: int = 0
|
||||
partial_tail_sent: bool = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self.token_len = 0
|
||||
@@ -720,6 +721,7 @@ class RequestTracker:
|
||||
self.token_ids = None
|
||||
self.has_pending_offload = False
|
||||
self.prefill_end_tokens = 0
|
||||
self.partial_tail_sent = False
|
||||
|
||||
def update(
|
||||
self,
|
||||
@@ -739,6 +741,14 @@ class RequestTracker:
|
||||
existing.extend(new)
|
||||
|
||||
|
||||
class BoundaryPut(NamedTuple):
|
||||
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
|
||||
|
||||
group_id: int
|
||||
block_id: int
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
"""Per-request metadata for store put/get operations."""
|
||||
@@ -756,15 +766,24 @@ class ReqMeta:
|
||||
# Absolute request offset represented by token_ids[0].
|
||||
token_ids_start: int = 0
|
||||
num_prompt_tokens: int | None = None
|
||||
# Total prefill tokens, including generated tokens replayed after preemption.
|
||||
prefill_end_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Core-provided (group_id, block_id, boundary_tokens) mamba "align"
|
||||
# boundary states. A block-aligned entry is a committed boundary snapshot;
|
||||
# a non-aligned entry is the sub-block CoW tail. The store-job reference
|
||||
# keeps each exact block alive until every worker rank finishes the job.
|
||||
boundary_state_offloads: list[tuple[int, int, int]] | None = None
|
||||
# Blocks stored under an explicit prefix hash rather than by position:
|
||||
# core-provided mamba "align" states (a block-aligned entry is a committed
|
||||
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
|
||||
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
|
||||
# store-job reference keeps each block alive until every worker rank
|
||||
# finishes the job.
|
||||
boundary_puts: list[BoundaryPut] | None = None
|
||||
# Total computed prefix length at the end of this step.
|
||||
completed_token_len: int | None = None
|
||||
# Set on the save that first covers the whole prompt, so the scheduler
|
||||
# publishes the prompt's partial tail once rather than on every save.
|
||||
publish_partial_tail: bool = False
|
||||
|
||||
@staticmethod
|
||||
def from_request_tracker(
|
||||
@@ -773,8 +792,12 @@ class ReqMeta:
|
||||
load_spec: LoadSpec | None = None,
|
||||
skip_save: bool | None = False,
|
||||
block_hashes: list[BlockHash] | None = None,
|
||||
save_partial_tail: bool = False,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> "ReqMeta | None":
|
||||
"""Create ReqMeta from a RequestTracker."""
|
||||
if save_partial_tail:
|
||||
assert num_prompt_tokens is not None
|
||||
if block_hashes is None:
|
||||
block_hashes = []
|
||||
input_token_len = tracker.token_len
|
||||
@@ -783,7 +806,15 @@ class ReqMeta:
|
||||
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
|
||||
num_tokens_to_save = input_token_len // block_size * block_size
|
||||
|
||||
skip_save = skip_save or num_tokens_to_save < chunk_boundary
|
||||
publish_tail = (
|
||||
save_partial_tail
|
||||
and not tracker.partial_tail_sent
|
||||
and num_prompt_tokens is not None
|
||||
and 0 < num_prompt_tokens <= input_token_len
|
||||
)
|
||||
skip_save = skip_save or (
|
||||
num_tokens_to_save < chunk_boundary and not publish_tail
|
||||
)
|
||||
# A ReqMeta must never carry both a save AND a load.
|
||||
# The save would also be wasted work — the bytes are being looked up
|
||||
# in the store right now. Later cached_reqs steps save new tokens
|
||||
@@ -795,6 +826,7 @@ class ReqMeta:
|
||||
|
||||
if not skip_save:
|
||||
tracker.num_saved_tokens = num_tokens_to_save
|
||||
tracker.partial_tail_sent |= publish_tail
|
||||
|
||||
token_ids = None
|
||||
if tracker.token_ids and not skip_save:
|
||||
@@ -826,7 +858,10 @@ class ReqMeta:
|
||||
block_hashes=block_hashes,
|
||||
token_ids=token_ids,
|
||||
token_ids_start=token_ids_start,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
completed_token_len=input_token_len,
|
||||
publish_partial_tail=publish_tail and not skip_save,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,17 @@
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Scheduler-side logic for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
partial_hash_hits_enabled,
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
BoundaryPut,
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
@@ -23,12 +26,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -49,6 +59,79 @@ def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
|
||||
return request.prompt_token_ids
|
||||
|
||||
|
||||
def _partial_tail_non_mamba_puts(
|
||||
coord: MooncakeStoreCoordinator,
|
||||
req_meta: ReqMeta,
|
||||
block_sizes: Sequence[int],
|
||||
) -> list[BoundaryPut]:
|
||||
"""Locate the non-Mamba blocks a partial-tail save publishes.
|
||||
|
||||
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
|
||||
if both pieces of KV are stored:
|
||||
|
||||
- Mamba groups: the state at ``boundary``, handed off by the core as a
|
||||
boundary put of its own, so Mamba groups are not listed here.
|
||||
- Other groups: the KV from the last LCM-aligned normal save up to
|
||||
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
|
||||
block is keyed by the hash at its end, clipped to ``proof_end``.
|
||||
|
||||
Groups whose proof is not computed yet are omitted. For example, with LCM
|
||||
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
|
||||
are 8-10, keyed at 36, 40 and 44.
|
||||
|
||||
A Mamba state handed off at any other position, i.e. a shared-prefix
|
||||
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
|
||||
|
||||
``block_sizes`` are the per-group block sizes, indexed like
|
||||
``req_meta.block_ids``.
|
||||
"""
|
||||
mamba_tails = [
|
||||
put.num_tokens
|
||||
for put in req_meta.boundary_puts or []
|
||||
if put.group_id in coord.mamba_group_ids
|
||||
and put.num_tokens % block_sizes[put.group_id]
|
||||
]
|
||||
# The tail is due on the save that first covers the prompt, or when Mamba
|
||||
# hands off its checkpoint state.
|
||||
if not mamba_tails and not req_meta.publish_partial_tail:
|
||||
return []
|
||||
prompt_tokens = req_meta.num_prompt_tokens
|
||||
completed = req_meta.completed_token_len
|
||||
assert prompt_tokens is not None and completed is not None
|
||||
hash_block_size = coord.hash_block_size
|
||||
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
|
||||
# groups carry the eagle flag.
|
||||
boundary = get_mamba_prefill_checkpoint_position(
|
||||
prompt_tokens, hash_block_size, coord.use_eagle
|
||||
)
|
||||
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
|
||||
# attention KV, EAGLE proof included, ends on an attention block end no later
|
||||
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
|
||||
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
|
||||
return []
|
||||
num_hashes = len(req_meta.block_hashes)
|
||||
if boundary == 0 or boundary // hash_block_size > num_hashes:
|
||||
return []
|
||||
start = boundary // coord.lcm_block_size * coord.lcm_block_size
|
||||
puts: list[BoundaryPut] = []
|
||||
for group_id, block_size in enumerate(block_sizes):
|
||||
if group_id in coord.mamba_group_ids:
|
||||
continue
|
||||
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
|
||||
if proof_end > completed or proof_end // hash_block_size > num_hashes:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[group_id]
|
||||
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
|
||||
if block_idx >= len(group_blocks):
|
||||
break
|
||||
block_id = group_blocks[block_idx]
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
num_tokens = min((block_idx + 1) * block_size, proof_end)
|
||||
puts.append(BoundaryPut(group_id, block_id, num_tokens))
|
||||
return puts
|
||||
|
||||
|
||||
class MooncakeStoreScheduler:
|
||||
"""Scheduler-side component for MooncakeStoreConnector."""
|
||||
|
||||
@@ -84,11 +167,6 @@ class MooncakeStoreScheduler:
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
store_groups,
|
||||
self._hash_block_size,
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
mamba_groups = {
|
||||
group_id: group.kv_cache_spec
|
||||
for group_id, group in enumerate(store_groups)
|
||||
@@ -99,6 +177,11 @@ class MooncakeStoreScheduler:
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._boundary_state_group_ids = frozenset(mamba_groups)
|
||||
|
||||
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
|
||||
)
|
||||
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
self._next_store_job_id = 0
|
||||
@@ -273,6 +356,8 @@ class MooncakeStoreScheduler:
|
||||
# producer. Loads are still carried by the same metadata.
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
@@ -323,6 +408,8 @@ class MooncakeStoreScheduler:
|
||||
load_spec=load_spec,
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
else:
|
||||
# Decode/chunked request
|
||||
@@ -364,6 +451,8 @@ class MooncakeStoreScheduler:
|
||||
load_spec=None,
|
||||
skip_save=False,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
|
||||
if req_meta is not None:
|
||||
@@ -398,6 +487,7 @@ class MooncakeStoreScheduler:
|
||||
load_spec=load_spec,
|
||||
skip_save=None,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
@@ -455,6 +545,8 @@ class MooncakeStoreScheduler:
|
||||
itself is freed, until every rank reports the job done.
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
coord = self._store_coord
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
continue
|
||||
@@ -464,27 +556,27 @@ class MooncakeStoreScheduler:
|
||||
if req_meta.store_job_id is not None:
|
||||
assert req_meta.store_job_id in self._pinned_saves
|
||||
continue
|
||||
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
|
||||
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids: list[int] = []
|
||||
if req_meta.boundary_state_offloads:
|
||||
block_ids.extend(
|
||||
block_id for _, block_id, _ in req_meta.boundary_state_offloads
|
||||
)
|
||||
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
|
||||
assert NULL_BLOCK_ID not in block_ids, (
|
||||
"A null block cannot back a boundary-state offload"
|
||||
)
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids.extend(
|
||||
block_id
|
||||
for group_id, group in enumerate(req_meta.block_ids)
|
||||
if group_id not in self._boundary_state_group_ids
|
||||
for block_id in group
|
||||
if block_id != NULL_BLOCK_ID
|
||||
"A null block cannot back a boundary put"
|
||||
)
|
||||
if req_meta.token_len_chunk:
|
||||
# Normal prefix save: pin all non-Mamba sources for retries.
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids.extend(
|
||||
block_id
|
||||
for group_id, group in enumerate(req_meta.block_ids)
|
||||
if group_id not in self._boundary_state_group_ids
|
||||
for block_id in group
|
||||
if block_id != NULL_BLOCK_ID
|
||||
)
|
||||
# An aligned boundary block may also be present in the request's
|
||||
# block table. Take and release exactly one reference per block.
|
||||
block_ids = list(dict.fromkeys(block_ids))
|
||||
@@ -515,17 +607,39 @@ class MooncakeStoreScheduler:
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
return False
|
||||
|
||||
pinned_block_ids: list[int] = []
|
||||
remapped_offloads: list[tuple[int, int, int]] = []
|
||||
mamba_puts: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary in partial_tail_offloads:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
return False
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
return False
|
||||
pinned_block_ids.append(block_id)
|
||||
remapped_offloads.append((store_group_id, block_id, boundary))
|
||||
pinned_block_ids = list(dict.fromkeys(pinned_block_ids))
|
||||
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
|
||||
req_meta = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(
|
||||
block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=mamba_puts,
|
||||
completed_token_len=request.num_computed_tokens,
|
||||
)
|
||||
coord = self._store_coord
|
||||
req_meta.boundary_puts = [
|
||||
*_partial_tail_non_mamba_puts(
|
||||
coord,
|
||||
req_meta,
|
||||
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
|
||||
),
|
||||
*mamba_puts,
|
||||
]
|
||||
pinned_block_ids = list(
|
||||
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
|
||||
)
|
||||
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None, (
|
||||
@@ -537,18 +651,8 @@ class MooncakeStoreScheduler:
|
||||
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
|
||||
|
||||
self._finished_partial_tail_metas[request.request_id] = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(
|
||||
block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
store_job_id=store_job_id,
|
||||
boundary_state_offloads=remapped_offloads,
|
||||
)
|
||||
req_meta.store_job_id = store_job_id
|
||||
self._finished_partial_tail_metas[request.request_id] = req_meta
|
||||
tracker.has_pending_offload = True
|
||||
# The store job owns exact block refs, so request cleanup need not wait.
|
||||
return False
|
||||
@@ -575,7 +679,7 @@ class MooncakeStoreScheduler:
|
||||
# going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping boundary-state offload for request %s", req_id)
|
||||
continue
|
||||
accepted: list[tuple[int, int, int]] = []
|
||||
accepted: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary_tokens in entries:
|
||||
# Every other group stops saving at the end of this prefill, so
|
||||
# a mamba-only key past it can never complete a joint hybrid
|
||||
@@ -589,12 +693,12 @@ class MooncakeStoreScheduler:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
continue
|
||||
accepted.append((store_group_id, block_id, boundary_tokens))
|
||||
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
|
||||
if not accepted:
|
||||
continue
|
||||
tracker.has_pending_offload = True
|
||||
if (req_meta := save_metas.get(req_id)) is not None:
|
||||
req_meta.boundary_state_offloads = accepted
|
||||
req_meta.boundary_puts = accepted
|
||||
continue
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
@@ -603,8 +707,10 @@ class MooncakeStoreScheduler:
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
boundary_state_offloads=accepted,
|
||||
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=accepted,
|
||||
completed_token_len=tracker.token_len,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -74,7 +74,6 @@ from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
resolve_dcp_kv_cache_spec,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -569,40 +568,43 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._skip_store_requests.clear()
|
||||
return True
|
||||
|
||||
def _boundary_snapshot_puts(
|
||||
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
|
||||
def _boundary_puts(
|
||||
self, req_meta: ReqMeta
|
||||
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
|
||||
"""Puts for committed mamba "align" boundary-state snapshots.
|
||||
"""Puts for ``req_meta.boundary_puts``: each block under the hash of
|
||||
the prefix ending at its ``num_tokens``.
|
||||
|
||||
These are block-aligned boundaries, i.e. exactly what the normal save
|
||||
would key — but ``store_mask`` masks mamba groups out of it entirely, so
|
||||
this is their *only* writer. The exclusion is not an optimization: the
|
||||
normal save resolves a chunk's address as
|
||||
``req_meta.block_ids[g][start // block_size]``, and ``block_ids`` is the
|
||||
connector's append-only mirror of the core's per-group table. An
|
||||
align-mode table is mutated in place (a superseded state block is freed
|
||||
and nulled; speculative blocks relocate), and the connector is never
|
||||
told, so a stale mirror entry is indistinguishable from a live one — a
|
||||
retry of a failed or pressure-skipped chunk would read a block that now
|
||||
belongs to another request.
|
||||
|
||||
Each entry's handed-off block *is* the boundary state and is pinned by
|
||||
the core, so it is uploaded under its boundary-end hash key and never
|
||||
resolved positionally.
|
||||
The scheduler resolves and pins every block, so none is looked up
|
||||
positionally here. That matters for mamba "align" states: ``store_mask``
|
||||
masks mamba groups out of the normal save, which resolves a chunk's
|
||||
address as ``req_meta.block_ids[g][start // block_size]``. An align-mode
|
||||
table is mutated in place (a superseded state block is freed and
|
||||
nulled; speculative blocks relocate) without telling the connector, so
|
||||
a retried positional read could hit a block that now belongs to another
|
||||
request. The handed-off block *is* the state.
|
||||
"""
|
||||
hash_block_size = self.coord.hash_block_size
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for group_id, block_id, boundary in entries:
|
||||
if boundary == 0 or block_id == NULL_BLOCK_ID:
|
||||
for group_id, block_id, num_tokens in req_meta.boundary_puts or []:
|
||||
if (
|
||||
not self.group_participates[group_id]
|
||||
or num_tokens == 0
|
||||
or block_id == NULL_BLOCK_ID
|
||||
):
|
||||
continue
|
||||
hash_idx = boundary // hash_block_size - 1
|
||||
# A negative index would silently key the block by the last hash.
|
||||
assert num_tokens % hash_block_size == 0, (
|
||||
f"Boundary put at {num_tokens} tokens is not a multiple of the "
|
||||
f"hash block size {hash_block_size}"
|
||||
)
|
||||
hash_idx = num_tokens // hash_block_size - 1
|
||||
if hash_idx >= len(req_meta.block_hashes):
|
||||
continue
|
||||
db = self.token_databases[group_id]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[group_id]
|
||||
put_step_rank = (self.tp_rank + group_id) % put_step
|
||||
if (boundary // db.block_size - 1) % put_step != put_step_rank:
|
||||
if (cdiv(num_tokens, db.block_size) - 1) % put_step != put_step_rank:
|
||||
continue
|
||||
addr, size = db.prepare_value_for_block(block_id)
|
||||
puts.append(
|
||||
@@ -610,115 +612,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
return puts
|
||||
|
||||
def _sub_block_tail_puts(
|
||||
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
|
||||
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
|
||||
"""Puts for the request's sub-block partial tail (its last prompt hash
|
||||
boundary), so a later request can hit the sub-block prefix.
|
||||
|
||||
Covers every group's blocks from the normal save's lcm floor to the
|
||||
boundary: the normal save floors to ``lcm_block_size``, so a
|
||||
smaller-block group's full blocks in that gap are never persisted
|
||||
elsewhere, and the consumer's lookup needs every group at every probed
|
||||
boundary. Full blocks are keyed by their block-end hash and the partial
|
||||
boundary block by the boundary sub-hash; a mamba "align" group
|
||||
contributes only its boundary block, from the core-provided CoW block.
|
||||
"""
|
||||
boundaries = {boundary for _, _, boundary in entries}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Sub-block partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary = boundaries.pop()
|
||||
hash_block_size = self.coord.hash_block_size
|
||||
if boundary == 0 or boundary // hash_block_size - 1 >= len(
|
||||
req_meta.block_hashes
|
||||
):
|
||||
return []
|
||||
|
||||
mamba_offloads = {group_id: block_id for group_id, block_id, _ in entries}
|
||||
saved = self._saved_offset.get(req_meta.req_id, 0)
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
if not self.group_participates[g_idx]:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[g_idx]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
# Always include the boundary block: its sub-hash key is written
|
||||
# only here, even if normal saves already advanced past it.
|
||||
last_block = cdiv(boundary, db.block_size) - 1
|
||||
for block_idx in range(
|
||||
min(saved // db.block_size, last_block), last_block + 1
|
||||
):
|
||||
if block_idx % put_step != put_step_rank:
|
||||
continue
|
||||
valid_end = min((block_idx + 1) * db.block_size, boundary)
|
||||
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
|
||||
if g_idx in mamba_offloads:
|
||||
if valid_end != boundary:
|
||||
# Interior align-mode state positions are null or
|
||||
# stale (the block table is not append-only) and never
|
||||
# valid gap content; only the boundary block is
|
||||
# persisted, from the core-provided hand-off.
|
||||
continue
|
||||
block_id = mamba_offloads[g_idx]
|
||||
elif g_idx in self.coord.mamba_group_ids:
|
||||
continue
|
||||
elif block_idx < len(group_blocks):
|
||||
block_id = group_blocks[block_idx]
|
||||
else:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
logger.debug(
|
||||
"Skipping unavailable partial-tail source block "
|
||||
"(req=%s, group=%d, block=%d)",
|
||||
req_meta.req_id,
|
||||
g_idx,
|
||||
block_idx,
|
||||
)
|
||||
continue
|
||||
addr, size = db.prepare_value_for_block(block_id)
|
||||
puts.append((db.key_for(key_hash), addr, size, db.metadata))
|
||||
return puts
|
||||
|
||||
def _maybe_offload_boundary_states(self, req_meta: ReqMeta) -> bool:
|
||||
"""Persist connector-pinned mamba "align" boundary states handed off
|
||||
for this request, deduped against the store.
|
||||
"""Persist this job's boundary puts, deduped against the store.
|
||||
|
||||
This is every mamba key the connector writes — ``store_mask`` excludes
|
||||
mamba groups from the positional normal save, aligned boundaries
|
||||
included (see :meth:`_boundary_snapshot_puts`).
|
||||
|
||||
The two entry kinds are keyed and sourced differently, so they are
|
||||
prepared separately and put in one batch:
|
||||
|
||||
- block-aligned for its group: a committed boundary-state snapshot,
|
||||
the handed-off block itself;
|
||||
- not block-aligned: the sub-block CoW partial tail, which also has to
|
||||
cover the other groups' blocks in the normal save's lcm gap.
|
||||
These are every mamba key the connector writes (``store_mask`` excludes
|
||||
mamba groups from the positional normal save) plus the other groups'
|
||||
partial-tail blocks in the normal save's lcm gap.
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
|
||||
"""
|
||||
offloads = req_meta.boundary_state_offloads
|
||||
if not offloads or not req_meta.block_hashes:
|
||||
if not req_meta.boundary_puts or not req_meta.block_hashes:
|
||||
return True
|
||||
|
||||
snapshots: list[tuple[int, int, int]] = []
|
||||
sub_block: list[tuple[int, int, int]] = []
|
||||
for group_id, block_id, boundary in offloads:
|
||||
entry = (group_id, block_id, boundary)
|
||||
if boundary % self.token_databases[group_id].block_size == 0:
|
||||
snapshots.append(entry)
|
||||
else:
|
||||
sub_block.append(entry)
|
||||
|
||||
puts = self._boundary_snapshot_puts(req_meta, snapshots)
|
||||
if sub_block and self.coord.enable_partial_hash_hits:
|
||||
puts.extend(self._sub_block_tail_puts(req_meta, sub_block))
|
||||
puts = self._boundary_puts(req_meta)
|
||||
|
||||
if not puts:
|
||||
return True
|
||||
@@ -854,9 +761,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
# Offload the handed-off mamba boundary states (independent of the
|
||||
# normal positional save, which may be skipped this step).
|
||||
if req_meta.boundary_state_offloads is not None and not (
|
||||
self._maybe_offload_boundary_states(req_meta)
|
||||
):
|
||||
if not self._maybe_offload_boundary_states(req_meta):
|
||||
return
|
||||
|
||||
if token_len == 0:
|
||||
@@ -870,7 +775,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
store_masks = self.coord.store_mask(
|
||||
token_len,
|
||||
save_start,
|
||||
num_prompt_tokens=req_meta.num_prompt_tokens,
|
||||
num_prompt_tokens=req_meta.prefill_end_tokens,
|
||||
)
|
||||
|
||||
starts: list[int] = []
|
||||
@@ -1603,31 +1508,10 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return
|
||||
|
||||
self._kv_cache_groups = [
|
||||
dataclasses.replace(
|
||||
group,
|
||||
kv_cache_spec=resolve_dcp_kv_cache_spec(
|
||||
group.kv_cache_spec,
|
||||
self.dcp_size,
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.prefix_cacheable_groups
|
||||
]
|
||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||
use_eagle_block_drop = bool(
|
||||
spec_cfg.use_eagle_block_drop()
|
||||
if spec_cfg is not None
|
||||
and callable(getattr(spec_cfg, "use_eagle_block_drop", None))
|
||||
else False
|
||||
)
|
||||
self.coord = MooncakeStoreCoordinator(
|
||||
self._kv_cache_groups,
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
use_eagle=use_eagle_block_drop,
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
dcp_world_size=self.dcp_size,
|
||||
self.coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self.block_size, self.hash_block_size
|
||||
)
|
||||
self._kv_cache_groups = self.coord.kv_cache_groups
|
||||
self.store_tp_size, store_namespace, store_layout_cls = (
|
||||
self._select_store_layout(extra_config)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user