[BugFix] Handle per-group prefix-hit divergence for hybrid models with KV connector (#48425)

This commit is contained in:
Nick Hill
2026-07-22 17:19:11 -07:00
committed by GitHub
parent 191146dba5
commit 229e01e9e1
4 changed files with 279 additions and 40 deletions
+197
View File
@@ -25,14 +25,17 @@ from vllm.multimodal.inputs import (
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
)
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
@@ -5464,3 +5467,197 @@ def test_async_load_reservation_prevents_wedge_e2e():
assert b.status == RequestStatus.WAITING
assert b.num_preemptions == 0
assert b.request_id not in req_to_blocks
def _create_hybrid_mamba_connector_scheduler(
matched_tokens: int,
block_size: int = 16,
num_blocks: int = 100,
) -> Scheduler:
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
model_config = ModelConfig(
model="facebook/opt-125m",
trust_remote_code=True,
dtype="float16",
seed=42,
skip_tokenizer_init=True,
)
vllm_config = VllmConfig(
scheduler_config=SchedulerConfig(
max_num_seqs=4,
max_num_batched_tokens=8192,
max_model_len=8192,
enable_chunked_prefill=True,
is_encoder_decoder=False,
watermark=0.0,
),
model_config=model_config,
cache_config=CacheConfig(
block_size=block_size,
enable_prefix_caching=True,
mamba_cache_mode="all",
),
kv_transfer_config=KVTransferConfig(
kv_connector="MockKVConnector",
kv_role="kv_both",
kv_connector_extra_config={
"matched_tokens": matched_tokens,
"is_async": False,
},
),
)
vllm_config.cache_config.num_gpu_blocks = num_blocks
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["fa"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
),
),
],
)
register_all_kvcache_specs(vllm_config)
return Scheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=block_size,
log_stats=True,
)
@pytest.mark.parametrize(
"matched_tokens,expected_num_computed",
[
# No external hit: resume on the deepest locally-consistent boundary
# (block 0's state survives for both groups).
(0, 16),
# One external block on top of the reconciled local boundary.
(16, 32),
],
)
def test_hybrid_per_group_hit_divergence_with_connector(
matched_tokens: int, expected_num_computed: int
):
"""Per-group prefix hits can diverge for hybrid models with a connector
(#46453): under block pressure the FA prefix tail is evicted while a
deeper Mamba state block survives. The scheduler must not report the
deeper hit as locally computed (evicted FA blocks are not resident ->
engine crash / dirty KV); it falls back to the reconciled boundary that
every group is consistent at.
"""
block_size = 16
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix so both groups cache all four boundaries
# (mamba cache mode "all" caches every block's state densely).
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
fa_ids = [b.block_id for b in blocks.blocks[0]]
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
# Evict the FA tail and the middle mamba states; block 0 (both groups)
# and the deep mamba state at block 3 survive.
manager.block_pool.evict_blocks({fa_ids[2], fa_ids[3], mamba_ids[1], mamba_ids[2]})
# A replay of the prefix plus one extra block now sees diverged
# per-group hits: FA stops at the evicted tail, while the mamba lookup
# finds the deeper surviving state.
[replay] = create_requests(
num_requests=1,
num_tokens=5 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["replay"],
)
_, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group(
replay.block_hashes, replay.num_tokens - 1
)
assert per_group_hits == (2 * block_size, 4 * block_size) # diverged
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
"""The opposite divergence: the FA prefix survives deeper than the Mamba
state and the connector supplies nothing (ext == 0). Reporting the deep FA
hit as locally computed would resume with no valid Mamba state at that
boundary (silent bad output). The scheduler must fall back to the
convergent boundary that every group agrees on (block 0's surviving state).
"""
block_size = 16
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0)
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix in both groups.
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
manager.block_pool.evict_blocks({mamba_ids[1], mamba_ids[2], mamba_ids[3]})
[replay] = create_requests(
num_requests=1,
num_tokens=5 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["replay"],
)
_, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group(
replay.block_hashes, replay.num_tokens - 1
)
assert per_group_hits == (4 * block_size, 1 * block_size) # FA deeper
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
# Must resume at the convergent boundary (block 0), not the deep FA hit.
assert replay.num_tokens - num_scheduled == block_size
+9
View File
@@ -634,6 +634,15 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
key=lambda g: not isinstance(g.spec, FullAttentionSpec)
)
# Dense reference group for per-group lookups (None when the model
# has no full-attention layers): full attention is downward-closed,
# so any group reporting a longer per-group hit implies the union of
# per-group hits is not consistent at a single boundary (#46453).
first = self.attention_groups[0]
self.full_attention_group_id: int | None = (
first.group_ids[0] if isinstance(first.spec, FullAttentionSpec) else None
)
# Propagate the eagle bit to each manager (default to ``use_eagle=False``).
for group in self.attention_groups:
if group.use_eagle:
+51 -1
View File
@@ -9,7 +9,10 @@ from typing import Literal, overload
from vllm.distributed.kv_events import BlockStored, KVCacheEvent
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator
from vllm.v1.core.kv_cache_coordinator import (
HybridKVCacheCoordinator,
get_kv_cache_coordinator,
)
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
from vllm.v1.kv_cache_interface import (
@@ -287,6 +290,53 @@ class KVCacheManager:
blocks = self.create_kv_cache_blocks(computed_blocks)
return blocks, num_new_computed_tokens, shared_prefix_boundary
def get_computed_blocks_for_connector(
self, request: Request
) -> tuple[KVCacheBlocks, int, int, bool]:
"""Local prefix-cache lookup for a request scheduled with a KV connector.
Hybrid (Mamba + full-attention) models can have per-group prefix hits
diverge under block pressure: the full-attention tail may be evicted
while a deeper Mamba state survives, or vice versa. Report the
full-attention hit as the local prefix - the connector transfers the
remaining suffix and the Mamba state is transferred unconditionally by
nixl's ``_apply_prefix_caching`` - and flag when that hit ran deeper
than a lagging group. Such a hit only has a valid Mamba state at its
boundary if the connector supplies it, so the caller must fall back to
``get_computed_blocks`` to reconcile when no external tokens are found.
Non-hybrid models and already-convergent hits use ``get_computed_blocks``.
Returns:
The ``get_computed_blocks`` triple (blocks, number of local computed
tokens, shared-prefix boundary) plus ``hit_diverged``.
"""
coordinator = self.coordinator
if not (
self.kv_cache_config.has_mamba_layers
and isinstance(coordinator, HybridKVCacheCoordinator)
and coordinator.full_attention_group_id is not None
):
return *self.get_computed_blocks(request), False
if not self.prefix_cache_lookup_enabled(request):
return self.empty_kv_cache_blocks, 0, 0, False
fa_group_id = coordinator.full_attention_group_id
computed, per_group_hits = coordinator.find_longest_cache_hit_per_group(
request.block_hashes, request.num_tokens - 1
)
if any(hit > per_group_hits[fa_group_id] for hit in per_group_hits):
# A lagging group hit deeper than full attention means its
# full-attention blocks were evicted; use the reconciled boundary
# that every group agrees on.
return *self.get_computed_blocks(request), False
num_local = per_group_hits[fa_group_id]
blocks = self.create_kv_cache_blocks(computed)
# Per-group lookups do not detect an uncached shared prefix (boundary 0).
return blocks, num_local, 0, min(per_group_hits) < num_local
def allocate_slots(
self,
request: Request,
+22 -39
View File
@@ -35,7 +35,6 @@ from vllm.v1.core.encoder_cache_manager import (
EncoderCacheManager,
EncoderDecoderCacheManager,
)
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import KVCacheBlock
@@ -711,50 +710,24 @@ class Scheduler(SchedulerInterface):
# Get already-cached tokens.
if request.num_computed_tokens == 0:
did_prefix_cache_lookup = True
hit_diverged = False
# Get locally-cached tokens.
if (
self.connector is not None
and self.has_mamba_layers
and isinstance(
self.kv_cache_manager.coordinator, HybridKVCacheCoordinator
if self.connector is not None:
# A KV connector transfers the missing suffix, which needs a
# hybrid-aware lookup that can diverge across groups.
(
new_computed_blocks,
num_new_local_computed_tokens,
request.shared_prefix_boundary,
hit_diverged,
) = self.kv_cache_manager.get_computed_blocks_for_connector(
request
)
):
# The per-group lookup does not detect an uncached shared
# prefix, so there is no junction to pin in this path.
request.shared_prefix_boundary = 0
kv_cache_manager = self.kv_cache_manager
if not kv_cache_manager.prefix_cache_lookup_enabled(request):
# Mirror the get_computed_blocks() early-out: the
# request must recompute its prompt.
new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks
num_new_local_computed_tokens = 0
else:
computed, per_group_hits = (
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
request.block_hashes, request.num_tokens - 1
)
)
new_computed_blocks = (
self.kv_cache_manager.create_kv_cache_blocks(computed)
)
# NOTE(ZhanqiuHu): For Mamba hybrid models,
# num_new_local_computed_tokens should be the FA hit
# length. This value is passed to the connector's
# get_num_new_matched_tokens which computes:
# external = total - local_computed.
# Using the FA hit skips re-transferring FA blocks
# already cached on D-side. The Mamba state (always
# the last block) is transferred unconditionally by
# _apply_prefix_caching in nixl/worker.py.
num_new_local_computed_tokens = max(per_group_hits)
else:
(
new_computed_blocks,
num_new_local_computed_tokens,
# Junction to pin (Marconi-style APC) so its
# sparse-retention state (Mamba block / sliding-window
# tail) survives retention and serves a later hit; 0
# if no uncached shared prefix was detected.
# Marconi shared-prefix junction to pin; 0 if none.
request.shared_prefix_boundary,
) = self.kv_cache_manager.get_computed_blocks(request)
@@ -776,6 +749,16 @@ class Scheduler(SchedulerInterface):
num_external_computed_tokens = ext_tokens
if hit_diverged and num_external_computed_tokens == 0:
# No external tokens back the deeper local hit, so its
# resume boundary would have no valid Mamba state.
# Reconcile to the boundary every group agrees on.
(
new_computed_blocks,
num_new_local_computed_tokens,
request.shared_prefix_boundary,
) = self.kv_cache_manager.get_computed_blocks(request)
connector_prefix_cache_queries = (
request.num_tokens - num_new_local_computed_tokens
)