[BugFix] Handle per-group prefix-hit divergence for hybrid models with KV connector (#48425)
This commit is contained in:
@@ -25,14 +25,17 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -5464,3 +5467,197 @@ def test_async_load_reservation_prevents_wedge_e2e():
|
||||
assert b.status == RequestStatus.WAITING
|
||||
assert b.num_preemptions == 0
|
||||
assert b.request_id not in req_to_blocks
|
||||
|
||||
|
||||
def _create_hybrid_mamba_connector_scheduler(
|
||||
matched_tokens: int,
|
||||
block_size: int = 16,
|
||||
num_blocks: int = 100,
|
||||
) -> Scheduler:
|
||||
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
trust_remote_code=True,
|
||||
dtype="float16",
|
||||
seed=42,
|
||||
skip_tokenizer_init=True,
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=8192,
|
||||
max_model_len=8192,
|
||||
enable_chunked_prefill=True,
|
||||
is_encoder_decoder=False,
|
||||
watermark=0.0,
|
||||
),
|
||||
model_config=model_config,
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="all",
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="MockKVConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"matched_tokens": matched_tokens,
|
||||
"is_async": False,
|
||||
},
|
||||
),
|
||||
)
|
||||
vllm_config.cache_config.num_gpu_blocks = num_blocks
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["fa"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
register_all_kvcache_specs(vllm_config)
|
||||
return Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=block_size,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matched_tokens,expected_num_computed",
|
||||
[
|
||||
# No external hit: resume on the deepest locally-consistent boundary
|
||||
# (block 0's state survives for both groups).
|
||||
(0, 16),
|
||||
# One external block on top of the reconciled local boundary.
|
||||
(16, 32),
|
||||
],
|
||||
)
|
||||
def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
matched_tokens: int, expected_num_computed: int
|
||||
):
|
||||
"""Per-group prefix hits can diverge for hybrid models with a connector
|
||||
(#46453): under block pressure the FA prefix tail is evicted while a
|
||||
deeper Mamba state block survives. The scheduler must not report the
|
||||
deeper hit as locally computed (evicted FA blocks are not resident ->
|
||||
engine crash / dirty KV); it falls back to the reconciled boundary that
|
||||
every group is consistent at.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries
|
||||
# (mamba cache mode "all" caches every block's state densely).
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
fa_ids = [b.block_id for b in blocks.blocks[0]]
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
|
||||
# Evict the FA tail and the middle mamba states; block 0 (both groups)
|
||||
# and the deep mamba state at block 3 survive.
|
||||
manager.block_pool.evict_blocks({fa_ids[2], fa_ids[3], mamba_ids[1], mamba_ids[2]})
|
||||
|
||||
# A replay of the prefix plus one extra block now sees diverged
|
||||
# per-group hits: FA stops at the evicted tail, while the mamba lookup
|
||||
# finds the deeper surviving state.
|
||||
[replay] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=5 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["replay"],
|
||||
)
|
||||
_, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group(
|
||||
replay.block_hashes, replay.num_tokens - 1
|
||||
)
|
||||
assert per_group_hits == (2 * block_size, 4 * block_size) # diverged
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
|
||||
|
||||
def test_hybrid_per_group_hit_divergence_fa_deeper_no_external():
|
||||
"""The opposite divergence: the FA prefix survives deeper than the Mamba
|
||||
state and the connector supplies nothing (ext == 0). Reporting the deep FA
|
||||
hit as locally computed would resume with no valid Mamba state at that
|
||||
boundary (silent bad output). The scheduler must fall back to the
|
||||
convergent boundary that every group agrees on (block 0's surviving state).
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens=0)
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix in both groups.
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
|
||||
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
|
||||
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
|
||||
manager.block_pool.evict_blocks({mamba_ids[1], mamba_ids[2], mamba_ids[3]})
|
||||
|
||||
[replay] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=5 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["replay"],
|
||||
)
|
||||
_, per_group_hits = manager.coordinator.find_longest_cache_hit_per_group(
|
||||
replay.block_hashes, replay.num_tokens - 1
|
||||
)
|
||||
assert per_group_hits == (4 * block_size, 1 * block_size) # FA deeper
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
# Must resume at the convergent boundary (block 0), not the deep FA hit.
|
||||
assert replay.num_tokens - num_scheduled == block_size
|
||||
|
||||
@@ -634,6 +634,15 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
key=lambda g: not isinstance(g.spec, FullAttentionSpec)
|
||||
)
|
||||
|
||||
# Dense reference group for per-group lookups (None when the model
|
||||
# has no full-attention layers): full attention is downward-closed,
|
||||
# so any group reporting a longer per-group hit implies the union of
|
||||
# per-group hits is not consistent at a single boundary (#46453).
|
||||
first = self.attention_groups[0]
|
||||
self.full_attention_group_id: int | None = (
|
||||
first.group_ids[0] if isinstance(first.spec, FullAttentionSpec) else None
|
||||
)
|
||||
|
||||
# Propagate the eagle bit to each manager (default to ``use_eagle=False``).
|
||||
for group in self.attention_groups:
|
||||
if group.use_eagle:
|
||||
|
||||
@@ -9,7 +9,10 @@ from typing import Literal, overload
|
||||
from vllm.distributed.kv_events import BlockStored, KVCacheEvent
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator
|
||||
from vllm.v1.core.kv_cache_coordinator import (
|
||||
HybridKVCacheCoordinator,
|
||||
get_kv_cache_coordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -287,6 +290,53 @@ class KVCacheManager:
|
||||
blocks = self.create_kv_cache_blocks(computed_blocks)
|
||||
return blocks, num_new_computed_tokens, shared_prefix_boundary
|
||||
|
||||
def get_computed_blocks_for_connector(
|
||||
self, request: Request
|
||||
) -> tuple[KVCacheBlocks, int, int, bool]:
|
||||
"""Local prefix-cache lookup for a request scheduled with a KV connector.
|
||||
|
||||
Hybrid (Mamba + full-attention) models can have per-group prefix hits
|
||||
diverge under block pressure: the full-attention tail may be evicted
|
||||
while a deeper Mamba state survives, or vice versa. Report the
|
||||
full-attention hit as the local prefix - the connector transfers the
|
||||
remaining suffix and the Mamba state is transferred unconditionally by
|
||||
nixl's ``_apply_prefix_caching`` - and flag when that hit ran deeper
|
||||
than a lagging group. Such a hit only has a valid Mamba state at its
|
||||
boundary if the connector supplies it, so the caller must fall back to
|
||||
``get_computed_blocks`` to reconcile when no external tokens are found.
|
||||
|
||||
Non-hybrid models and already-convergent hits use ``get_computed_blocks``.
|
||||
|
||||
Returns:
|
||||
The ``get_computed_blocks`` triple (blocks, number of local computed
|
||||
tokens, shared-prefix boundary) plus ``hit_diverged``.
|
||||
"""
|
||||
coordinator = self.coordinator
|
||||
if not (
|
||||
self.kv_cache_config.has_mamba_layers
|
||||
and isinstance(coordinator, HybridKVCacheCoordinator)
|
||||
and coordinator.full_attention_group_id is not None
|
||||
):
|
||||
return *self.get_computed_blocks(request), False
|
||||
|
||||
if not self.prefix_cache_lookup_enabled(request):
|
||||
return self.empty_kv_cache_blocks, 0, 0, False
|
||||
|
||||
fa_group_id = coordinator.full_attention_group_id
|
||||
computed, per_group_hits = coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
if any(hit > per_group_hits[fa_group_id] for hit in per_group_hits):
|
||||
# A lagging group hit deeper than full attention means its
|
||||
# full-attention blocks were evicted; use the reconciled boundary
|
||||
# that every group agrees on.
|
||||
return *self.get_computed_blocks(request), False
|
||||
|
||||
num_local = per_group_hits[fa_group_id]
|
||||
blocks = self.create_kv_cache_blocks(computed)
|
||||
# Per-group lookups do not detect an uncached shared prefix (boundary 0).
|
||||
return blocks, num_local, 0, min(per_group_hits) < num_local
|
||||
|
||||
def allocate_slots(
|
||||
self,
|
||||
request: Request,
|
||||
|
||||
@@ -35,7 +35,6 @@ from vllm.v1.core.encoder_cache_manager import (
|
||||
EncoderCacheManager,
|
||||
EncoderDecoderCacheManager,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
@@ -711,50 +710,24 @@ class Scheduler(SchedulerInterface):
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
did_prefix_cache_lookup = True
|
||||
hit_diverged = False
|
||||
# Get locally-cached tokens.
|
||||
if (
|
||||
self.connector is not None
|
||||
and self.has_mamba_layers
|
||||
and isinstance(
|
||||
self.kv_cache_manager.coordinator, HybridKVCacheCoordinator
|
||||
if self.connector is not None:
|
||||
# A KV connector transfers the missing suffix, which needs a
|
||||
# hybrid-aware lookup that can diverge across groups.
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
request.shared_prefix_boundary,
|
||||
hit_diverged,
|
||||
) = self.kv_cache_manager.get_computed_blocks_for_connector(
|
||||
request
|
||||
)
|
||||
):
|
||||
# The per-group lookup does not detect an uncached shared
|
||||
# prefix, so there is no junction to pin in this path.
|
||||
request.shared_prefix_boundary = 0
|
||||
kv_cache_manager = self.kv_cache_manager
|
||||
if not kv_cache_manager.prefix_cache_lookup_enabled(request):
|
||||
# Mirror the get_computed_blocks() early-out: the
|
||||
# request must recompute its prompt.
|
||||
new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks
|
||||
num_new_local_computed_tokens = 0
|
||||
else:
|
||||
computed, per_group_hits = (
|
||||
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
)
|
||||
new_computed_blocks = (
|
||||
self.kv_cache_manager.create_kv_cache_blocks(computed)
|
||||
)
|
||||
# NOTE(ZhanqiuHu): For Mamba hybrid models,
|
||||
# num_new_local_computed_tokens should be the FA hit
|
||||
# length. This value is passed to the connector's
|
||||
# get_num_new_matched_tokens which computes:
|
||||
# external = total - local_computed.
|
||||
# Using the FA hit skips re-transferring FA blocks
|
||||
# already cached on D-side. The Mamba state (always
|
||||
# the last block) is transferred unconditionally by
|
||||
# _apply_prefix_caching in nixl/worker.py.
|
||||
num_new_local_computed_tokens = max(per_group_hits)
|
||||
else:
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
# Junction to pin (Marconi-style APC) so its
|
||||
# sparse-retention state (Mamba block / sliding-window
|
||||
# tail) survives retention and serves a later hit; 0
|
||||
# if no uncached shared prefix was detected.
|
||||
# Marconi shared-prefix junction to pin; 0 if none.
|
||||
request.shared_prefix_boundary,
|
||||
) = self.kv_cache_manager.get_computed_blocks(request)
|
||||
|
||||
@@ -776,6 +749,16 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
num_external_computed_tokens = ext_tokens
|
||||
|
||||
if hit_diverged and num_external_computed_tokens == 0:
|
||||
# No external tokens back the deeper local hit, so its
|
||||
# resume boundary would have no valid Mamba state.
|
||||
# Reconcile to the boundary every group agrees on.
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
request.shared_prefix_boundary,
|
||||
) = self.kv_cache_manager.get_computed_blocks(request)
|
||||
|
||||
connector_prefix_cache_queries = (
|
||||
request.num_tokens - num_new_local_computed_tokens
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user