[Bugfix] OffloadingConnector: stop zeroing offload hits under MTP/EAGLE spec decode (#52771)

Signed-off-by: Kam Basra <kameldipbasra@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Kam Basra
2026-09-07 15:16:05 +03:00
committed by GitHub
co-authored by Claude
parent 94e26dd3dd
commit 4a806d08ee
3 changed files with 297 additions and 21 deletions
@@ -50,6 +50,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
@@ -3070,8 +3071,9 @@ class TestEagle:
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4.
This clears eagle_verified. Group 1 runs with max_hit=4 → only 1
key queried, 1 hit, pop to 0 → returns 0.
This clears eagle_verified. Group 1 runs with max_hit=4 → widened
query of 2 keys, 2 hits, pop to 1 → the confirmed boundary holds
at 4 tokens.
"""
block_size = 4
groups = [
@@ -3116,9 +3118,12 @@ class TestEagle:
offload_keys_per_group=[[10, 11, 12], [1, 2, 3]],
)
# Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4
# Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1].
# Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0
assert sched._lookup(req_status) == 0
# Group 1 (eagle FA): max_hit=4, widened query → keys=[1, 2].
# Finds 2 hits, pop to 1 → boundary stays at 4 tokens. Before the
# #52735 fix the query was not widened for full-attention groups,
# so the pop landed on the only queried chunk and zeroed the
# whole request.
assert sched._lookup(req_status) == 4
def test_eagle_verified_survives_eagle_tighten(self, request_runner):
"""Eagle group tightening does NOT clear eagle_verified.
@@ -3234,10 +3239,12 @@ class TestEagle:
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
# 4 decoded tokens fill block 3 entirely with decode tokens (one
# extra token so the block is stored under async scheduling too).
# 4 decoded tokens fill block 3; the extra decode steps let its store
# job complete within this run under both scheduling modes. The
# eagle group holds back its volatile trailing block (1, 3) while the
# request is still decoding, while the normal group stores (0, 3).
runner.run(
decoded_tokens=[1, 1, 1, 1, 1, EOS_TOKEN_ID],
decoded_tokens=[1, 1, 1, 1, 1, 1, 1],
expected_stored=(
(0, 0),
(0, 1),
@@ -3248,6 +3255,13 @@ class TestEagle:
(1, 2),
),
)
# Once the request finishes, no spec rejection can rewrite the tail,
# so the exclusion is lifted (issue #52735): the held-back (1, 3) and
# the just-completed block 4 are stored for both groups.
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=((1, 3),),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_sw_store_excludes_trailing_decode_block(
@@ -3289,11 +3303,17 @@ class TestEagle:
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
# 4 decoded tokens fill block 3 entirely with decode tokens.
# 4 decoded tokens fill block 3 entirely with decode tokens. The
# eagle group holds back its volatile trailing block while decoding.
runner.run(
decoded_tokens=[1, 1, 1, 1, EOS_TOKEN_ID],
decoded_tokens=[1, 1, 1, 1],
expected_stored=((0, 0), (0, 1), (0, 2)),
)
# Finish lifts the exclusion; the tail block is stored (issue #52735).
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=((0, 3),),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_single_block_stored_at_end_of_prefill(
@@ -3569,8 +3589,8 @@ class TestEagle:
)
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
scheduler.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
request = MagicMock()
@@ -4171,3 +4191,232 @@ def test_retention_interval_zero_stores_only_replay_boundary(
(1, 11),
),
)
def _shared_kv_mtp_config():
"""Speculative config for a shared-group MTP model: eagle-family method
whose drafter layer merges into a target KV-cache group, so no group
self-identifies as a drafter group (issue #52735)."""
spec = MagicMock(name="shared_kv_mtp_spec")
spec.use_eagle.return_value = True
spec.use_eagle_block_drop.return_value = True
spec.use_multi_module_mtp.return_value = False
spec.num_speculative_tokens_per_batch_size = None
spec.max_num_new_slots_for_drafting = 0
spec.num_speculative_tokens = 1
return spec
class TestSharedGroupMTPOffload:
"""Regression tests for issue #52735: OffloadingConnector must keep
serving when speculative decoding is enabled but no KV-cache group is
annotated as a drafter group (shared-group MTP models)."""
def test_no_annotation_marks_no_groups(self, request_runner):
"""Spec decode on + zero annotated groups must NOT mark every group
as a drafter group; the full store->load roundtrip must match the
non-speculative behavior of test_two_groups_full_and_sliding_window."""
block_size = 4
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=8,
),
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=True,
kv_cache_groups=kv_cache_groups,
speculative_config=_shared_kv_mtp_config(),
)
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
assert [c.is_eagle_group for c in kv_group_configs] == [False, False]
runner.new_request(token_ids=[0] * block_size * 3)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(decoded_tokens=[0])
runner.run(
decoded_tokens=[0] * (block_size * 3 + 2),
expected_stored=(0, 1, 2, 3, 4, 5),
)
runner.run(decoded_tokens=[EOS_TOKEN_ID])
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * (block_size * 3 + 1))
runner.manager.lookup.return_value = LookupResult.HIT
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
)
class TestMambaHybridOffloadServing:
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
with a manager that only HITs keys that were actually stored. Guards the
two collapse routes of issue #52735."""
BLOCK = 4
MAMBA_BLOCK = 16
PROMPT_TOKENS = 28 # 7 hash blocks; 1 full mamba chunk
def _make_scheduler(self, speculative_config, mamba_eagle=False):
vllm_config = _make_vllm_config(
extra_config={"self_describing_kv_events": True}
)
vllm_config.cache_config.block_size = self.BLOCK
vllm_config.cache_config.prefix_match_unit = self.BLOCK
vllm_config.speculative_config = speculative_config
vllm_config.kv_events_config = KVEventsConfig(
enable_kv_cache_events=True, publisher="null"
)
kv_cache_config = KVCacheConfig(
num_blocks=64,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full_layer"],
FullAttentionSpec(
block_size=self.BLOCK,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=mamba_eagle,
),
KVCacheGroupSpec(
["mamba_layer"],
MambaSpec(
block_size=self.MAMBA_BLOCK,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
def _roundtrip_served_tokens(self, scheduler) -> int:
"""Request A: prompt-only store, finish; request A2: served tokens."""
stored: set = set()
scheduler.manager.lookup.side_effect = lambda key, ctx: (
LookupResult.HIT if key in stored else LookupResult.MISS
)
scheduler.manager.prepare_store.side_effect = lambda keys, ctx: (
generate_store_output(list(keys))
)
def complete_all_jobs():
job_ids = list(scheduler._jobs.keys())
for jid in job_ids:
stored.update(scheduler._jobs[jid].keys)
if job_ids:
scheduler.update_connector_output(
KVConnectorOutput(
kv_connector_worker_meta=OffloadingWorkerMetadata(
completed_jobs={jid: 1 for jid in job_ids}
)
)
)
def make_request(req_id):
request = MagicMock()
request.request_id = req_id
request.kv_transfer_params = None
request.num_prompt_tokens = self.PROMPT_TOKENS
request.num_tokens = self.PROMPT_TOKENS
request.num_computed_tokens = 0
request.block_hashes = [
BlockHash(f"h{i}".encode())
for i in range(self.PROMPT_TOKENS // self.BLOCK)
]
request.all_token_ids = list(range(self.PROMPT_TOKENS))
request.lora_request = None
request.is_finished.return_value = False
request.status = None
scheduler.on_new_request(request)
return request
req = make_request("A")
req_status = scheduler._req_status["A"]
req_status.num_locally_computed_tokens = 0
req_status.update_offload_keys()
assert scheduler._lookup(req_status) == 0 # cold
n_full = self.PROMPT_TOKENS // self.BLOCK
n_mamba = self.PROMPT_TOKENS // self.MAMBA_BLOCK
req_status.group_states[0].block_ids[:] = list(range(1, n_full + 1))
req_status.group_states[1].block_ids[:] = list(range(101, 101 + n_mamba + 1))
out = SimpleNamespace(
num_scheduled_tokens={"A": self.PROMPT_TOKENS},
finished_req_ids=set(),
kv_connector_block_state=KVConnectorBlockState(
block_ids={},
boundary_state_offloads={"A": [(1, 101, self.MAMBA_BLOCK)]},
),
)
scheduler._build_partial_tail_store_jobs(out)
scheduler._build_store_jobs(out)
complete_all_jobs()
req.num_computed_tokens = self.PROMPT_TOKENS
req.num_tokens = self.PROMPT_TOKENS + 1
req.is_finished.return_value = True
req_status.update_offload_keys()
scheduler.request_finished(req)
out2 = SimpleNamespace(num_scheduled_tokens={}, finished_req_ids={"A"})
scheduler._build_store_jobs(out2)
complete_all_jobs()
make_request("A2")
rs2 = scheduler._req_status["A2"]
rs2.num_locally_computed_tokens = 0
rs2.update_offload_keys()
served = scheduler._lookup(rs2)
return served if served is not None else 0
def test_shared_group_mtp_serves_like_non_speculative(self):
"""Route 1 of #52735: with spec decode on and no drafter annotation,
serving must equal the non-speculative baseline (was 0 before the
fix: the all-groups fallback marked the mamba group as a drafter and
the volatile-tail pop consumed its only servable chunk)."""
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
assert baseline == 16
served = self._roundtrip_served_tokens(
self._make_scheduler(_shared_kv_mtp_config())
)
assert served == baseline
def test_annotated_eagle_group_does_not_starve_coarse_sibling(self):
"""Route 2 of #52735 (F1): a genuinely annotated eagle full-attention
group must not drag the confirmed boundary below a coarser sibling's
chunk granularity. The widened query makes the volatile-tail pop
land on an extra queried chunk, holding the boundary at 16 tokens
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
scheduler = self._make_scheduler(None, mamba_eagle=True)
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
True,
False,
]
assert self._roundtrip_served_tokens(scheduler) == 16
@@ -181,6 +181,7 @@ class RequestRunner:
extra_config_overrides: dict[str, Any] | None = None,
worker_count: int = 1,
retention_interval: int | None = None,
speculative_config: Any | None = None,
):
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
"blocks_per_chunk > 1 requires all groups to have the same "
@@ -202,6 +203,8 @@ class RequestRunner:
vllm_config.scheduler_config.async_scheduling = async_scheduling
vllm_config.parallel_config.world_size = worker_count
vllm_config.cache_config.prefix_cache_retention_interval = retention_interval
if speculative_config is not None:
vllm_config.speculative_config = speculative_config
extra_config: dict[str, Any] = {
"spec_name": "MockOffloadingSpec",
@@ -679,6 +682,7 @@ def request_runner():
extra_config_overrides=None,
worker_count=1,
retention_interval=None,
speculative_config=None,
):
runner = RequestRunner(
block_size=block_size,
@@ -689,6 +693,7 @@ def request_runner():
extra_config_overrides=extra_config_overrides,
worker_count=worker_count,
retention_interval=retention_interval,
speculative_config=speculative_config,
)
runners.append(runner)
return runner
@@ -208,7 +208,21 @@ class SchedulerOffloadConfig(NamedTuple):
and vllm_config.speculative_config.use_eagle_block_drop()
)
if use_eagle_block_drop and not eagle_groups:
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
# No group is annotated as holding drafter layers. This happens
# for shared-group MTP models (e.g. Qwen3.5-style), whose drafter
# is a regular decoder layer merged into the target's
# full-attention group, and for separate-draft models that miss
# annotation. Treating every group as a drafter group here makes
# the volatile-tail pop apply to ALL groups, which can zero the
# whole request's offload hit whenever any group's servable
# window is a single chunk (issue #52735). Drafter KV served one
# chunk stale can only lower speculative acceptance rates — the
# target model verifies every draft — so fail toward serving.
logger.info_once(
"KV offloading: speculative decoding is enabled but no "
"KV-cache group is annotated as a drafter group; treating "
"all groups as non-draft for offloading."
)
if eagle_groups:
logger.info(
@@ -395,15 +409,19 @@ class RequestOffloadState:
accepted position may be rewritten after spec-token rejection. During
prefill the trailing chunk is stable (the draft input for a chunk's
last position is the next prompt token), so it is stored immediately.
The exclusion must be applied consistently everywhere
``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of
each step is skipped on collection but jumped over by
``next_stored_chunk_idx``, so it is never re-considered and a
permanent hole breaks prefix-reuse lookup.
Once the request has finished, no further spec-token rejection can
rewrite the tail, so the exclusion is lifted and the final chunk
becomes storable (issue #52735). The exclusion must be applied
consistently everywhere ``next_stored_chunk_idx`` is derived:
otherwise the trailing chunk of each step is skipped on collection but
jumped over by ``next_stored_chunk_idx``, so it is never re-considered
and a permanent hole breaks prefix-reuse lookup. ``is_finished`` is
monotonic, so the finish-time calls all see the lifted exclusion.
"""
num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
if group_config.is_eagle_group and is_decoding:
# Finished requests have no pending speculation.
if group_config.is_eagle_group and is_decoding and not self.req.is_finished():
num_chunks = max(0, num_chunks - 1)
num_allocated_chunks = (
len(group_state.block_ids) // self.config.blocks_per_chunk
@@ -761,9 +779,13 @@ class OffloadingConnectorScheduler:
)
# For eagle groups, query one extra chunk that will be popped.
# We only need to increase the query size for sliding window groups.
# Widening applies to every group type: without it, the pop
# below shrinks max_hit_size_tokens past what was queried,
# which can push the confirmed boundary under a coarser
# sibling group's chunk granularity and zero the whole
# request's hit (issue #52735).
query_max = max_hit_size_tokens
if is_eagle_unverified and sliding_window_size_in_chunks is not None:
if is_eagle_unverified:
query_max = min(
max_hit_size_tokens + tokens_per_chunk,
len(offload_keys) * tokens_per_chunk,