From 4a806d08ee94b35356dd749bf814ee91ee93f8d0 Mon Sep 17 00:00:00 2001 From: Kam Basra Date: Mon, 7 Sep 2026 13:16:05 +0100 Subject: [PATCH] [Bugfix] OffloadingConnector: stop zeroing offload hits under MTP/EAGLE spec decode (#52771) Signed-off-by: Kam Basra Co-authored-by: Claude --- .../offloading_connector/test_scheduler.py | 273 +++++++++++++++++- .../unit/offloading_connector/utils.py | 5 + .../kv_connector/v1/offloading/scheduler.py | 40 ++- 3 files changed, 297 insertions(+), 21 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 2a1b599661..cd9fb1d4ee 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -50,6 +50,7 @@ from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, + MambaSpec, SlidingWindowSpec, ) from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry @@ -3070,8 +3071,9 @@ class TestEagle: Groups: 0=non-eagle full-attn, 1=eagle full-attn. Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4. - This clears eagle_verified. Group 1 runs with max_hit=4 → only 1 - key queried, 1 hit, pop to 0 → returns 0. + This clears eagle_verified. Group 1 runs with max_hit=4 → widened + query of 2 keys, 2 hits, pop to 1 → the confirmed boundary holds + at 4 tokens. """ block_size = 4 groups = [ @@ -3116,9 +3118,12 @@ class TestEagle: offload_keys_per_group=[[10, 11, 12], [1, 2, 3]], ) # Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4 - # Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1]. - # Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0 - assert sched._lookup(req_status) == 0 + # Group 1 (eagle FA): max_hit=4, widened query → keys=[1, 2]. + # Finds 2 hits, pop to 1 → boundary stays at 4 tokens. Before the + # #52735 fix the query was not widened for full-attention groups, + # so the pop landed on the only queried chunk and zeroed the + # whole request. + assert sched._lookup(req_status) == 4 def test_eagle_verified_survives_eagle_tighten(self, request_runner): """Eagle group tightening does NOT clear eagle_verified. @@ -3234,10 +3239,12 @@ class TestEagle: runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) - # 4 decoded tokens fill block 3 entirely with decode tokens (one - # extra token so the block is stored under async scheduling too). + # 4 decoded tokens fill block 3; the extra decode steps let its store + # job complete within this run under both scheduling modes. The + # eagle group holds back its volatile trailing block (1, 3) while the + # request is still decoding, while the normal group stores (0, 3). runner.run( - decoded_tokens=[1, 1, 1, 1, 1, EOS_TOKEN_ID], + decoded_tokens=[1, 1, 1, 1, 1, 1, 1], expected_stored=( (0, 0), (0, 1), @@ -3248,6 +3255,13 @@ class TestEagle: (1, 2), ), ) + # Once the request finishes, no spec rejection can rewrite the tail, + # so the exclusion is lifted (issue #52735): the held-back (1, 3) and + # the just-completed block 4 are stored for both groups. + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=((1, 3),), + ) @pytest.mark.parametrize("async_scheduling", [True, False]) def test_sw_store_excludes_trailing_decode_block( @@ -3289,11 +3303,17 @@ class TestEagle: runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) - # 4 decoded tokens fill block 3 entirely with decode tokens. + # 4 decoded tokens fill block 3 entirely with decode tokens. The + # eagle group holds back its volatile trailing block while decoding. runner.run( - decoded_tokens=[1, 1, 1, 1, EOS_TOKEN_ID], + decoded_tokens=[1, 1, 1, 1], expected_stored=((0, 0), (0, 1), (0, 2)), ) + # Finish lifts the exclusion; the tail block is stored (issue #52735). + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=((0, 3),), + ) @pytest.mark.parametrize("async_scheduling", [True, False]) def test_single_block_stored_at_end_of_prefill( @@ -3569,8 +3589,8 @@ class TestEagle: ) spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) - scheduler.manager.prepare_store.side_effect = ( - lambda keys, req_context: generate_store_output(keys) + scheduler.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) ) request = MagicMock() @@ -4171,3 +4191,232 @@ def test_retention_interval_zero_stores_only_replay_boundary( (1, 11), ), ) + + +def _shared_kv_mtp_config(): + """Speculative config for a shared-group MTP model: eagle-family method + whose drafter layer merges into a target KV-cache group, so no group + self-identifies as a drafter group (issue #52735).""" + spec = MagicMock(name="shared_kv_mtp_spec") + spec.use_eagle.return_value = True + spec.use_eagle_block_drop.return_value = True + spec.use_multi_module_mtp.return_value = False + spec.num_speculative_tokens_per_batch_size = None + spec.max_num_new_slots_for_drafting = 0 + spec.num_speculative_tokens = 1 + return spec + + +class TestSharedGroupMTPOffload: + """Regression tests for issue #52735: OffloadingConnector must keep + serving when speculative decoding is enabled but no KV-cache group is + annotated as a drafter group (shared-group MTP models).""" + + def test_no_annotation_marks_no_groups(self, request_runner): + """Spec decode on + zero annotated groups must NOT mark every group + as a drafter group; the full store->load roundtrip must match the + non-speculative behavior of test_two_groups_full_and_sliding_window.""" + block_size = 4 + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=8, + ), + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=True, + kv_cache_groups=kv_cache_groups, + speculative_config=_shared_kv_mtp_config(), + ) + kv_group_configs = runner.connector_scheduler.config.kv_group_configs + assert [c.is_eagle_group for c in kv_group_configs] == [False, False] + + runner.new_request(token_ids=[0] * block_size * 3) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.run(decoded_tokens=[0]) + runner.run( + decoded_tokens=[0] * (block_size * 3 + 2), + expected_stored=(0, 1, 2, 3, 4, 5), + ) + runner.run(decoded_tokens=[EOS_TOKEN_ID]) + + runner.scheduler.reset_prefix_cache() + + runner.new_request(token_ids=[0] * (block_size * 3 + 1)) + runner.manager.lookup.return_value = LookupResult.HIT + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), + ) + + +class TestMambaHybridOffloadServing: + """Store->finish->lookup flows on a full-attention + mamba-align hybrid, + with a manager that only HITs keys that were actually stored. Guards the + two collapse routes of issue #52735.""" + + BLOCK = 4 + MAMBA_BLOCK = 16 + PROMPT_TOKENS = 28 # 7 hash blocks; 1 full mamba chunk + + def _make_scheduler(self, speculative_config, mamba_eagle=False): + vllm_config = _make_vllm_config( + extra_config={"self_describing_kv_events": True} + ) + vllm_config.cache_config.block_size = self.BLOCK + vllm_config.cache_config.prefix_match_unit = self.BLOCK + vllm_config.speculative_config = speculative_config + vllm_config.kv_events_config = KVEventsConfig( + enable_kv_cache_events=True, publisher="null" + ) + kv_cache_config = KVCacheConfig( + num_blocks=64, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full_layer"], + FullAttentionSpec( + block_size=self.BLOCK, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=mamba_eagle, + ), + KVCacheGroupSpec( + ["mamba_layer"], + MambaSpec( + block_size=self.MAMBA_BLOCK, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + ], + ) + spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) + + def _roundtrip_served_tokens(self, scheduler) -> int: + """Request A: prompt-only store, finish; request A2: served tokens.""" + stored: set = set() + scheduler.manager.lookup.side_effect = lambda key, ctx: ( + LookupResult.HIT if key in stored else LookupResult.MISS + ) + scheduler.manager.prepare_store.side_effect = lambda keys, ctx: ( + generate_store_output(list(keys)) + ) + + def complete_all_jobs(): + job_ids = list(scheduler._jobs.keys()) + for jid in job_ids: + stored.update(scheduler._jobs[jid].keys) + if job_ids: + scheduler.update_connector_output( + KVConnectorOutput( + kv_connector_worker_meta=OffloadingWorkerMetadata( + completed_jobs={jid: 1 for jid in job_ids} + ) + ) + ) + + def make_request(req_id): + request = MagicMock() + request.request_id = req_id + request.kv_transfer_params = None + request.num_prompt_tokens = self.PROMPT_TOKENS + request.num_tokens = self.PROMPT_TOKENS + request.num_computed_tokens = 0 + request.block_hashes = [ + BlockHash(f"h{i}".encode()) + for i in range(self.PROMPT_TOKENS // self.BLOCK) + ] + request.all_token_ids = list(range(self.PROMPT_TOKENS)) + request.lora_request = None + request.is_finished.return_value = False + request.status = None + scheduler.on_new_request(request) + return request + + req = make_request("A") + req_status = scheduler._req_status["A"] + req_status.num_locally_computed_tokens = 0 + req_status.update_offload_keys() + assert scheduler._lookup(req_status) == 0 # cold + + n_full = self.PROMPT_TOKENS // self.BLOCK + n_mamba = self.PROMPT_TOKENS // self.MAMBA_BLOCK + req_status.group_states[0].block_ids[:] = list(range(1, n_full + 1)) + req_status.group_states[1].block_ids[:] = list(range(101, 101 + n_mamba + 1)) + + out = SimpleNamespace( + num_scheduled_tokens={"A": self.PROMPT_TOKENS}, + finished_req_ids=set(), + kv_connector_block_state=KVConnectorBlockState( + block_ids={}, + boundary_state_offloads={"A": [(1, 101, self.MAMBA_BLOCK)]}, + ), + ) + scheduler._build_partial_tail_store_jobs(out) + scheduler._build_store_jobs(out) + complete_all_jobs() + + req.num_computed_tokens = self.PROMPT_TOKENS + req.num_tokens = self.PROMPT_TOKENS + 1 + req.is_finished.return_value = True + req_status.update_offload_keys() + scheduler.request_finished(req) + out2 = SimpleNamespace(num_scheduled_tokens={}, finished_req_ids={"A"}) + scheduler._build_store_jobs(out2) + complete_all_jobs() + + make_request("A2") + rs2 = scheduler._req_status["A2"] + rs2.num_locally_computed_tokens = 0 + rs2.update_offload_keys() + served = scheduler._lookup(rs2) + return served if served is not None else 0 + + def test_shared_group_mtp_serves_like_non_speculative(self): + """Route 1 of #52735: with spec decode on and no drafter annotation, + serving must equal the non-speculative baseline (was 0 before the + fix: the all-groups fallback marked the mamba group as a drafter and + the volatile-tail pop consumed its only servable chunk).""" + baseline = self._roundtrip_served_tokens(self._make_scheduler(None)) + assert baseline == 16 + served = self._roundtrip_served_tokens( + self._make_scheduler(_shared_kv_mtp_config()) + ) + assert served == baseline + + def test_annotated_eagle_group_does_not_starve_coarse_sibling(self): + """Route 2 of #52735 (F1): a genuinely annotated eagle full-attention + group must not drag the confirmed boundary below a coarser sibling's + chunk granularity. The widened query makes the volatile-tail pop + land on an extra queried chunk, holding the boundary at 16 tokens + (was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk).""" + scheduler = self._make_scheduler(None, mamba_eagle=True) + assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [ + True, + False, + ] + assert self._roundtrip_served_tokens(scheduler) == 16 diff --git a/tests/v1/kv_connector/unit/offloading_connector/utils.py b/tests/v1/kv_connector/unit/offloading_connector/utils.py index f322ed7f1f..5707003ce9 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/utils.py +++ b/tests/v1/kv_connector/unit/offloading_connector/utils.py @@ -181,6 +181,7 @@ class RequestRunner: extra_config_overrides: dict[str, Any] | None = None, worker_count: int = 1, retention_interval: int | None = None, + speculative_config: Any | None = None, ): assert blocks_per_chunk == 1 or kv_cache_groups is None, ( "blocks_per_chunk > 1 requires all groups to have the same " @@ -202,6 +203,8 @@ class RequestRunner: vllm_config.scheduler_config.async_scheduling = async_scheduling vllm_config.parallel_config.world_size = worker_count vllm_config.cache_config.prefix_cache_retention_interval = retention_interval + if speculative_config is not None: + vllm_config.speculative_config = speculative_config extra_config: dict[str, Any] = { "spec_name": "MockOffloadingSpec", @@ -679,6 +682,7 @@ def request_runner(): extra_config_overrides=None, worker_count=1, retention_interval=None, + speculative_config=None, ): runner = RequestRunner( block_size=block_size, @@ -689,6 +693,7 @@ def request_runner(): extra_config_overrides=extra_config_overrides, worker_count=worker_count, retention_interval=retention_interval, + speculative_config=speculative_config, ) runners.append(runner) return runner diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 03a16edc9b..050aebe8f5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -208,7 +208,21 @@ class SchedulerOffloadConfig(NamedTuple): and vllm_config.speculative_config.use_eagle_block_drop() ) if use_eagle_block_drop and not eagle_groups: - eagle_groups = set(range(len(kv_cache_config.kv_cache_groups))) + # No group is annotated as holding drafter layers. This happens + # for shared-group MTP models (e.g. Qwen3.5-style), whose drafter + # is a regular decoder layer merged into the target's + # full-attention group, and for separate-draft models that miss + # annotation. Treating every group as a drafter group here makes + # the volatile-tail pop apply to ALL groups, which can zero the + # whole request's offload hit whenever any group's servable + # window is a single chunk (issue #52735). Drafter KV served one + # chunk stale can only lower speculative acceptance rates — the + # target model verifies every draft — so fail toward serving. + logger.info_once( + "KV offloading: speculative decoding is enabled but no " + "KV-cache group is annotated as a drafter group; treating " + "all groups as non-draft for offloading." + ) if eagle_groups: logger.info( @@ -395,15 +409,19 @@ class RequestOffloadState: accepted position may be rewritten after spec-token rejection. During prefill the trailing chunk is stable (the draft input for a chunk's last position is the next prompt token), so it is stored immediately. - The exclusion must be applied consistently everywhere - ``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of - each step is skipped on collection but jumped over by - ``next_stored_chunk_idx``, so it is never re-considered and a - permanent hole breaks prefix-reuse lookup. + Once the request has finished, no further spec-token rejection can + rewrite the tail, so the exclusion is lifted and the final chunk + becomes storable (issue #52735). The exclusion must be applied + consistently everywhere ``next_stored_chunk_idx`` is derived: + otherwise the trailing chunk of each step is skipped on collection but + jumped over by ``next_stored_chunk_idx``, so it is never re-considered + and a permanent hole breaks prefix-reuse lookup. ``is_finished`` is + monotonic, so the finish-time calls all see the lifted exclusion. """ num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens - if group_config.is_eagle_group and is_decoding: + # Finished requests have no pending speculation. + if group_config.is_eagle_group and is_decoding and not self.req.is_finished(): num_chunks = max(0, num_chunks - 1) num_allocated_chunks = ( len(group_state.block_ids) // self.config.blocks_per_chunk @@ -761,9 +779,13 @@ class OffloadingConnectorScheduler: ) # For eagle groups, query one extra chunk that will be popped. - # We only need to increase the query size for sliding window groups. + # Widening applies to every group type: without it, the pop + # below shrinks max_hit_size_tokens past what was queried, + # which can push the confirmed boundary under a coarser + # sibling group's chunk granularity and zero the whole + # request's hit (issue #52735). query_max = max_hit_size_tokens - if is_eagle_unverified and sliding_window_size_in_chunks is not None: + if is_eagle_unverified: query_max = min( max_hit_size_tokens + tokens_per_chunk, len(offload_keys) * tokens_per_chunk,