[Bugfix] OffloadingConnector: stop zeroing offload hits under MTP/EAGLE spec decode (#52771)
Signed-off-by: Kam Basra <kameldipbasra@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -50,6 +50,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
@@ -3070,8 +3071,9 @@ class TestEagle:
|
||||
|
||||
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
|
||||
Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4.
|
||||
This clears eagle_verified. Group 1 runs with max_hit=4 → only 1
|
||||
key queried, 1 hit, pop to 0 → returns 0.
|
||||
This clears eagle_verified. Group 1 runs with max_hit=4 → widened
|
||||
query of 2 keys, 2 hits, pop to 1 → the confirmed boundary holds
|
||||
at 4 tokens.
|
||||
"""
|
||||
block_size = 4
|
||||
groups = [
|
||||
@@ -3116,9 +3118,12 @@ class TestEagle:
|
||||
offload_keys_per_group=[[10, 11, 12], [1, 2, 3]],
|
||||
)
|
||||
# Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4
|
||||
# Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1].
|
||||
# Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0
|
||||
assert sched._lookup(req_status) == 0
|
||||
# Group 1 (eagle FA): max_hit=4, widened query → keys=[1, 2].
|
||||
# Finds 2 hits, pop to 1 → boundary stays at 4 tokens. Before the
|
||||
# #52735 fix the query was not widened for full-attention groups,
|
||||
# so the pop landed on the only queried chunk and zeroed the
|
||||
# whole request.
|
||||
assert sched._lookup(req_status) == 4
|
||||
|
||||
def test_eagle_verified_survives_eagle_tighten(self, request_runner):
|
||||
"""Eagle group tightening does NOT clear eagle_verified.
|
||||
@@ -3234,10 +3239,12 @@ class TestEagle:
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
# 4 decoded tokens fill block 3 entirely with decode tokens (one
|
||||
# extra token so the block is stored under async scheduling too).
|
||||
# 4 decoded tokens fill block 3; the extra decode steps let its store
|
||||
# job complete within this run under both scheduling modes. The
|
||||
# eagle group holds back its volatile trailing block (1, 3) while the
|
||||
# request is still decoding, while the normal group stores (0, 3).
|
||||
runner.run(
|
||||
decoded_tokens=[1, 1, 1, 1, 1, EOS_TOKEN_ID],
|
||||
decoded_tokens=[1, 1, 1, 1, 1, 1, 1],
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
@@ -3248,6 +3255,13 @@ class TestEagle:
|
||||
(1, 2),
|
||||
),
|
||||
)
|
||||
# Once the request finishes, no spec rejection can rewrite the tail,
|
||||
# so the exclusion is lifted (issue #52735): the held-back (1, 3) and
|
||||
# the just-completed block 4 are stored for both groups.
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=((1, 3),),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_sw_store_excludes_trailing_decode_block(
|
||||
@@ -3289,11 +3303,17 @@ class TestEagle:
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
# 4 decoded tokens fill block 3 entirely with decode tokens.
|
||||
# 4 decoded tokens fill block 3 entirely with decode tokens. The
|
||||
# eagle group holds back its volatile trailing block while decoding.
|
||||
runner.run(
|
||||
decoded_tokens=[1, 1, 1, 1, EOS_TOKEN_ID],
|
||||
decoded_tokens=[1, 1, 1, 1],
|
||||
expected_stored=((0, 0), (0, 1), (0, 2)),
|
||||
)
|
||||
# Finish lifts the exclusion; the tail block is stored (issue #52735).
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=((0, 3),),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_single_block_stored_at_end_of_prefill(
|
||||
@@ -3569,8 +3589,8 @@ class TestEagle:
|
||||
)
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
scheduler.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
request = MagicMock()
|
||||
@@ -4171,3 +4191,232 @@ def test_retention_interval_zero_stores_only_replay_boundary(
|
||||
(1, 11),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _shared_kv_mtp_config():
|
||||
"""Speculative config for a shared-group MTP model: eagle-family method
|
||||
whose drafter layer merges into a target KV-cache group, so no group
|
||||
self-identifies as a drafter group (issue #52735)."""
|
||||
spec = MagicMock(name="shared_kv_mtp_spec")
|
||||
spec.use_eagle.return_value = True
|
||||
spec.use_eagle_block_drop.return_value = True
|
||||
spec.use_multi_module_mtp.return_value = False
|
||||
spec.num_speculative_tokens_per_batch_size = None
|
||||
spec.max_num_new_slots_for_drafting = 0
|
||||
spec.num_speculative_tokens = 1
|
||||
return spec
|
||||
|
||||
|
||||
class TestSharedGroupMTPOffload:
|
||||
"""Regression tests for issue #52735: OffloadingConnector must keep
|
||||
serving when speculative decoding is enabled but no KV-cache group is
|
||||
annotated as a drafter group (shared-group MTP models)."""
|
||||
|
||||
def test_no_annotation_marks_no_groups(self, request_runner):
|
||||
"""Spec decode on + zero annotated groups must NOT mark every group
|
||||
as a drafter group; the full store->load roundtrip must match the
|
||||
non-speculative behavior of test_two_groups_full_and_sliding_window."""
|
||||
block_size = 4
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
),
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=True,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
speculative_config=_shared_kv_mtp_config(),
|
||||
)
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert [c.is_eagle_group for c in kv_group_configs] == [False, False]
|
||||
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (block_size * 3 + 2),
|
||||
expected_stored=(0, 1, 2, 3, 4, 5),
|
||||
)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
runner.new_request(token_ids=[0] * (block_size * 3 + 1))
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||
)
|
||||
|
||||
|
||||
class TestMambaHybridOffloadServing:
|
||||
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
|
||||
with a manager that only HITs keys that were actually stored. Guards the
|
||||
two collapse routes of issue #52735."""
|
||||
|
||||
BLOCK = 4
|
||||
MAMBA_BLOCK = 16
|
||||
PROMPT_TOKENS = 28 # 7 hash blocks; 1 full mamba chunk
|
||||
|
||||
def _make_scheduler(self, speculative_config, mamba_eagle=False):
|
||||
vllm_config = _make_vllm_config(
|
||||
extra_config={"self_describing_kv_events": True}
|
||||
)
|
||||
vllm_config.cache_config.block_size = self.BLOCK
|
||||
vllm_config.cache_config.prefix_match_unit = self.BLOCK
|
||||
vllm_config.speculative_config = speculative_config
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="null"
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full_layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=self.BLOCK,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=mamba_eagle,
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba_layer"],
|
||||
MambaSpec(
|
||||
block_size=self.MAMBA_BLOCK,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
|
||||
def _roundtrip_served_tokens(self, scheduler) -> int:
|
||||
"""Request A: prompt-only store, finish; request A2: served tokens."""
|
||||
stored: set = set()
|
||||
scheduler.manager.lookup.side_effect = lambda key, ctx: (
|
||||
LookupResult.HIT if key in stored else LookupResult.MISS
|
||||
)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, ctx: (
|
||||
generate_store_output(list(keys))
|
||||
)
|
||||
|
||||
def complete_all_jobs():
|
||||
job_ids = list(scheduler._jobs.keys())
|
||||
for jid in job_ids:
|
||||
stored.update(scheduler._jobs[jid].keys)
|
||||
if job_ids:
|
||||
scheduler.update_connector_output(
|
||||
KVConnectorOutput(
|
||||
kv_connector_worker_meta=OffloadingWorkerMetadata(
|
||||
completed_jobs={jid: 1 for jid in job_ids}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def make_request(req_id):
|
||||
request = MagicMock()
|
||||
request.request_id = req_id
|
||||
request.kv_transfer_params = None
|
||||
request.num_prompt_tokens = self.PROMPT_TOKENS
|
||||
request.num_tokens = self.PROMPT_TOKENS
|
||||
request.num_computed_tokens = 0
|
||||
request.block_hashes = [
|
||||
BlockHash(f"h{i}".encode())
|
||||
for i in range(self.PROMPT_TOKENS // self.BLOCK)
|
||||
]
|
||||
request.all_token_ids = list(range(self.PROMPT_TOKENS))
|
||||
request.lora_request = None
|
||||
request.is_finished.return_value = False
|
||||
request.status = None
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
|
||||
req = make_request("A")
|
||||
req_status = scheduler._req_status["A"]
|
||||
req_status.num_locally_computed_tokens = 0
|
||||
req_status.update_offload_keys()
|
||||
assert scheduler._lookup(req_status) == 0 # cold
|
||||
|
||||
n_full = self.PROMPT_TOKENS // self.BLOCK
|
||||
n_mamba = self.PROMPT_TOKENS // self.MAMBA_BLOCK
|
||||
req_status.group_states[0].block_ids[:] = list(range(1, n_full + 1))
|
||||
req_status.group_states[1].block_ids[:] = list(range(101, 101 + n_mamba + 1))
|
||||
|
||||
out = SimpleNamespace(
|
||||
num_scheduled_tokens={"A": self.PROMPT_TOKENS},
|
||||
finished_req_ids=set(),
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"A": [(1, 101, self.MAMBA_BLOCK)]},
|
||||
),
|
||||
)
|
||||
scheduler._build_partial_tail_store_jobs(out)
|
||||
scheduler._build_store_jobs(out)
|
||||
complete_all_jobs()
|
||||
|
||||
req.num_computed_tokens = self.PROMPT_TOKENS
|
||||
req.num_tokens = self.PROMPT_TOKENS + 1
|
||||
req.is_finished.return_value = True
|
||||
req_status.update_offload_keys()
|
||||
scheduler.request_finished(req)
|
||||
out2 = SimpleNamespace(num_scheduled_tokens={}, finished_req_ids={"A"})
|
||||
scheduler._build_store_jobs(out2)
|
||||
complete_all_jobs()
|
||||
|
||||
make_request("A2")
|
||||
rs2 = scheduler._req_status["A2"]
|
||||
rs2.num_locally_computed_tokens = 0
|
||||
rs2.update_offload_keys()
|
||||
served = scheduler._lookup(rs2)
|
||||
return served if served is not None else 0
|
||||
|
||||
def test_shared_group_mtp_serves_like_non_speculative(self):
|
||||
"""Route 1 of #52735: with spec decode on and no drafter annotation,
|
||||
serving must equal the non-speculative baseline (was 0 before the
|
||||
fix: the all-groups fallback marked the mamba group as a drafter and
|
||||
the volatile-tail pop consumed its only servable chunk)."""
|
||||
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
|
||||
assert baseline == 16
|
||||
served = self._roundtrip_served_tokens(
|
||||
self._make_scheduler(_shared_kv_mtp_config())
|
||||
)
|
||||
assert served == baseline
|
||||
|
||||
def test_annotated_eagle_group_does_not_starve_coarse_sibling(self):
|
||||
"""Route 2 of #52735 (F1): a genuinely annotated eagle full-attention
|
||||
group must not drag the confirmed boundary below a coarser sibling's
|
||||
chunk granularity. The widened query makes the volatile-tail pop
|
||||
land on an extra queried chunk, holding the boundary at 16 tokens
|
||||
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
|
||||
scheduler = self._make_scheduler(None, mamba_eagle=True)
|
||||
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert self._roundtrip_served_tokens(scheduler) == 16
|
||||
|
||||
@@ -181,6 +181,7 @@ class RequestRunner:
|
||||
extra_config_overrides: dict[str, Any] | None = None,
|
||||
worker_count: int = 1,
|
||||
retention_interval: int | None = None,
|
||||
speculative_config: Any | None = None,
|
||||
):
|
||||
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
|
||||
"blocks_per_chunk > 1 requires all groups to have the same "
|
||||
@@ -202,6 +203,8 @@ class RequestRunner:
|
||||
vllm_config.scheduler_config.async_scheduling = async_scheduling
|
||||
vllm_config.parallel_config.world_size = worker_count
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = retention_interval
|
||||
if speculative_config is not None:
|
||||
vllm_config.speculative_config = speculative_config
|
||||
|
||||
extra_config: dict[str, Any] = {
|
||||
"spec_name": "MockOffloadingSpec",
|
||||
@@ -679,6 +682,7 @@ def request_runner():
|
||||
extra_config_overrides=None,
|
||||
worker_count=1,
|
||||
retention_interval=None,
|
||||
speculative_config=None,
|
||||
):
|
||||
runner = RequestRunner(
|
||||
block_size=block_size,
|
||||
@@ -689,6 +693,7 @@ def request_runner():
|
||||
extra_config_overrides=extra_config_overrides,
|
||||
worker_count=worker_count,
|
||||
retention_interval=retention_interval,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
runners.append(runner)
|
||||
return runner
|
||||
|
||||
@@ -208,7 +208,21 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
and vllm_config.speculative_config.use_eagle_block_drop()
|
||||
)
|
||||
if use_eagle_block_drop and not eagle_groups:
|
||||
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
|
||||
# No group is annotated as holding drafter layers. This happens
|
||||
# for shared-group MTP models (e.g. Qwen3.5-style), whose drafter
|
||||
# is a regular decoder layer merged into the target's
|
||||
# full-attention group, and for separate-draft models that miss
|
||||
# annotation. Treating every group as a drafter group here makes
|
||||
# the volatile-tail pop apply to ALL groups, which can zero the
|
||||
# whole request's offload hit whenever any group's servable
|
||||
# window is a single chunk (issue #52735). Drafter KV served one
|
||||
# chunk stale can only lower speculative acceptance rates — the
|
||||
# target model verifies every draft — so fail toward serving.
|
||||
logger.info_once(
|
||||
"KV offloading: speculative decoding is enabled but no "
|
||||
"KV-cache group is annotated as a drafter group; treating "
|
||||
"all groups as non-draft for offloading."
|
||||
)
|
||||
|
||||
if eagle_groups:
|
||||
logger.info(
|
||||
@@ -395,15 +409,19 @@ class RequestOffloadState:
|
||||
accepted position may be rewritten after spec-token rejection. During
|
||||
prefill the trailing chunk is stable (the draft input for a chunk's
|
||||
last position is the next prompt token), so it is stored immediately.
|
||||
The exclusion must be applied consistently everywhere
|
||||
``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of
|
||||
each step is skipped on collection but jumped over by
|
||||
``next_stored_chunk_idx``, so it is never re-considered and a
|
||||
permanent hole breaks prefix-reuse lookup.
|
||||
Once the request has finished, no further spec-token rejection can
|
||||
rewrite the tail, so the exclusion is lifted and the final chunk
|
||||
becomes storable (issue #52735). The exclusion must be applied
|
||||
consistently everywhere ``next_stored_chunk_idx`` is derived:
|
||||
otherwise the trailing chunk of each step is skipped on collection but
|
||||
jumped over by ``next_stored_chunk_idx``, so it is never re-considered
|
||||
and a permanent hole breaks prefix-reuse lookup. ``is_finished`` is
|
||||
monotonic, so the finish-time calls all see the lifted exclusion.
|
||||
"""
|
||||
num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
|
||||
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
|
||||
if group_config.is_eagle_group and is_decoding:
|
||||
# Finished requests have no pending speculation.
|
||||
if group_config.is_eagle_group and is_decoding and not self.req.is_finished():
|
||||
num_chunks = max(0, num_chunks - 1)
|
||||
num_allocated_chunks = (
|
||||
len(group_state.block_ids) // self.config.blocks_per_chunk
|
||||
@@ -761,9 +779,13 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
|
||||
# For eagle groups, query one extra chunk that will be popped.
|
||||
# We only need to increase the query size for sliding window groups.
|
||||
# Widening applies to every group type: without it, the pop
|
||||
# below shrinks max_hit_size_tokens past what was queried,
|
||||
# which can push the confirmed boundary under a coarser
|
||||
# sibling group's chunk granularity and zero the whole
|
||||
# request's hit (issue #52735).
|
||||
query_max = max_hit_size_tokens
|
||||
if is_eagle_unverified and sliding_window_size_in_chunks is not None:
|
||||
if is_eagle_unverified:
|
||||
query_max = min(
|
||||
max_hit_size_tokens + tokens_per_chunk,
|
||||
len(offload_keys) * tokens_per_chunk,
|
||||
|
||||
Reference in New Issue
Block a user