From 7ad894c86a2f3615fe72d739c25567803b5924ec Mon Sep 17 00:00:00 2001 From: joshua abraham <132982099+JOSH1024@users.noreply.github.com> Date: Tue, 16 Jun 2026 13:28:39 +0530 Subject: [PATCH] [Bugfix] Prevent cuMemcpyBatchAsync segfault with MTP and KV offloading (#44784) Signed-off-by: joshua Co-authored-by: joshua Co-authored-by: Or Ozeri --- .../offloading_connector/test_scheduler.py | 682 ++++++++++++++++++ .../kv_connector/v1/offloading/scheduler.py | 69 +- 2 files changed, 745 insertions(+), 6 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 11da73b315..8bd46184d6 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -14,6 +14,7 @@ from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID from vllm.distributed.kv_events import BlockRemoved, BlockStored from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( OffloadingConnectorScheduler, + RequestOffloadState, ) from vllm.v1.core.kv_cache_utils import BlockHash from vllm.v1.kv_cache_interface import ( @@ -28,6 +29,7 @@ from vllm.v1.kv_offload.base import ( ReqContext, RequestOffloadingContext, get_offload_block_hash, + make_offload_key, ) from vllm.v1.request import RequestStatus @@ -1432,3 +1434,683 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool): # The external lookup must have been completely skipped. runner.manager.lookup.assert_not_called() + + +# --------------------------------------------------------------------------- +# Eagle/MTP test class +# --------------------------------------------------------------------------- + + +class TestEagle: + """Tests for Eagle/MTP speculative decoding support in the offloading + connector scheduler — both _lookup() unit tests and integration tests.""" + + # ------------------------------------------------------------------- + # Helpers + # ------------------------------------------------------------------- + + @staticmethod + def _group_keys(group_idx: int, int_hashes: list[int]) -> list: + return [make_offload_key(str(h).encode(), group_idx) for h in int_hashes] + + @staticmethod + def _make_req_status( + scheduler: OffloadingConnectorScheduler, + *, + num_tokens: int, + num_computed_tokens: int = 0, + offload_keys_per_group: list[list[int]], + ) -> RequestOffloadState: + """Build RequestOffloadState with synthetic offload keys.""" + req = MagicMock() + req.request_id = "test-req" + req.num_tokens = num_tokens + req.kv_transfer_params = None + + state = RequestOffloadState( + config=scheduler.config, + req=req, + req_context=ReqContext(req_id="test-req"), + offloading_context=RequestOffloadingContext( + policy=OffloadPolicy.BLOCK_LEVEL + ), + num_locally_computed_tokens=num_computed_tokens, + ) + for idx, (gs, hashes) in enumerate( + zip(state.group_states, offload_keys_per_group) + ): + gs.offload_keys = TestEagle._group_keys( + scheduler.config.kv_group_configs[idx].group_idx, hashes + ) + return state + + # ------------------------------------------------------------------- + # Lookup unit tests: call _lookup() directly via request_runner + # ------------------------------------------------------------------- + + def test_full_attn_lookup_pops_one_block(self, request_runner): + """Full-attn eagle group with 3 blocks all hit → pop to 2 blocks.""" + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2, 3} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, num_tokens=12, offload_keys_per_group=[[1, 2, 3]] + ) + # 3 hits, pop to 2 → 2 * block_size = 8 tokens loadable + assert sched._lookup(req_status) == 8 + + def test_full_attn_lookup_single_block_returns_zero(self, request_runner): + """Full-attn eagle group with 1 block hit → pop to 0 → returns 0.""" + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, num_tokens=4, offload_keys_per_group=[[1]] + ) + # 1 hit, pop to 0 → new_num_hit_tokens < block_size → return 0 + assert sched._lookup(req_status) == 0 + + def test_full_attn_lookup_no_hits_returns_zero(self, request_runner): + """Full-attn eagle group with 0 hits returns 0 before pop.""" + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.return_value = False + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, num_tokens=8, offload_keys_per_group=[[1, 2]] + ) + assert sched._lookup(req_status) == 0 + + def test_sw_lookup_inflates_query_max(self, request_runner): + """SW eagle group inflates query_max so _sliding_window_lookup gets + one extra key beyond what max_hit_size_tokens alone would yield. + + With block_size=4, W=2, eagle, num_tokens=13, 4 keys all hitting: + - max_hit = 13-1 = 12 (SW reduction) + - Without inflation: num_blocks = cdiv(12,4) = 3 → only 3 keys + - With inflation: query_max = min(12+4, 4*4=16) = 16, + num_blocks = cdiv(16,4) = 4 → 4 keys passed to SW + - SW finds window of 3 (required=W+1=3) at idx 1 → returns 4 + - Pop: 4-1=3 → max_hit = min(12, 12) = 12. Result: 12. + """ + block_size = 4 + sw_blocks = 2 + groups = [ + KVCacheGroupSpec( + ["layer0"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sw_blocks * block_size, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2, 3, 4} + ) + sched = runner.connector_scheduler + + captured_keys: list = [] + orig_sw_lookup = type(sched)._sliding_window_lookup + + def capturing_sw_lookup(self_arg, keys, window, req_context): + captured_keys.append(list(keys)) + return orig_sw_lookup(self_arg, keys, window, req_context) + + sched._sliding_window_lookup = lambda keys, window, req_ctx: ( + capturing_sw_lookup(sched, keys, window, req_ctx) + ) + + req_status = self._make_req_status( + sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3, 4]] + ) + result = sched._lookup(req_status) + assert len(captured_keys) == 1 + # Inflation bumped from 3 keys (cdiv(12,4)) to 4 keys (cdiv(16,4)) + assert len(captured_keys[0]) == 4 + # SW finds window of 3 → returns 4, pop to 3 → 3*4=12 + assert result == 12 + + def test_sw_lookup_requires_extra_window_block(self, request_runner): + """SW eagle with W=2 and only 2 keys (both hit) uses prefix fallback. + + Since required_window = W+1 = 3 but only 2 keys are available + (inflation is capped by len(offload_keys)), _sliding_window_lookup + can never find a window of 3. It falls back to prefix count (2). + Pop: 2-1=1 → max_hit = 4. Result: 4 tokens (degraded from full hit). + """ + block_size = 4 + sw_blocks = 2 + groups = [ + KVCacheGroupSpec( + ["layer0"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sw_blocks * block_size, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, num_tokens=9, offload_keys_per_group=[[1, 2]] + ) + # Prefix fallback returns 2, pop to 1 → 1*4 = 4 tokens + assert sched._lookup(req_status) == 4 + + def test_sw_lookup_w_plus_one_hits_returns_w_blocks(self, request_runner): + """SW eagle with W=2, 3 contiguous hits → pop to 2 → returns 2*bs.""" + block_size = 4 + sw_blocks = 2 + groups = [ + KVCacheGroupSpec( + ["layer0"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sw_blocks * block_size, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2, 3} + ) + sched = runner.connector_scheduler + # num_tokens=13 → max_hit=13-1=12, query_max=min(12+4,12)=12 + # num_blocks=cdiv(12,4)=3, keys=[1,2,3], required_window=3 + # SW finds window of 3, pop to 2 → 2*4=8 + req_status = self._make_req_status( + sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3]] + ) + assert sched._lookup(req_status) == 8 + + def test_eagle_verified_prevents_double_pop(self, request_runner): + """Once an eagle group has popped, it doesn't pop again on re-iteration. + + Setup: group 0 = non-eagle full-attn (3 blocks), group 1 = eagle + full-attn (3 blocks). Both see all hits. Eagle pops to 2 and tightens + max_hit to 8. Group 0 re-runs (convergence) but since eagle_verified + contains group 1, it won't pop again — result stays at 8 tokens. + """ + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=False, + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2, 3} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, + num_tokens=12, + offload_keys_per_group=[[1, 2, 3], [1, 2, 3]], + ) + # Group 0: prefix finds 3 → max_hit=12, num_hit=12 + # Group 1 (eagle): prefix finds 3, pop to 2 → max_hit=8, num_hit=8 + # num_hit(8) < prev num_hit(12) AND group IS eagle → no clear + # No re-iteration triggered (eagle shrink doesn't trigger re-loop) + # Final: 8 tokens + assert sched._lookup(req_status) == 8 + + def test_non_eagle_tighten_clears_eagle_verified(self, request_runner): + """Non-eagle group tightening clears eagle_verified → eagle re-pops. + + Groups: 0=non-eagle full-attn, 1=eagle full-attn. + Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4. + This clears eagle_verified. Group 1 runs with max_hit=4 → only 1 + key queried, 1 hit, pop to 0 → returns 0. + """ + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=False, + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + # Group 0 keys [10,11,12]: only 10 hits. + # Group 1 keys [1,2,3]: all hit. + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {10, 1, 2, 3} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, + num_tokens=12, + offload_keys_per_group=[[10, 11, 12], [1, 2, 3]], + ) + # Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4 + # Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1]. + # Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0 + assert sched._lookup(req_status) == 0 + + def test_eagle_verified_survives_eagle_tighten(self, request_runner): + """Eagle group tightening does NOT clear eagle_verified. + + Groups: 0=non-eagle full-attn, 1=eagle full-attn. + Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2 + (max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared. + Result: 8 tokens (eagle only pops once). + """ + block_size = 4 + groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=False, + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + runner = request_runner( + block_size=block_size, + num_gpu_blocks=100, + async_scheduling=False, + kv_cache_groups=groups, + ) + runner.manager.lookup.side_effect = lambda key, req_context: ( + int(get_offload_block_hash(key).decode()) in {1, 2, 3} + ) + sched = runner.connector_scheduler + req_status = self._make_req_status( + sched, + num_tokens=12, + offload_keys_per_group=[[1, 2, 3], [1, 2, 3]], + ) + # Group 0: 3 hits → max_hit=12, num_hit=12 + # Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8 + # Tightened but IS eagle → no clear. No re-iteration. + assert sched._lookup(req_status) == 8 + + # ------------------------------------------------------------------- + # Integration tests: store and load via request_runner + # ------------------------------------------------------------------- + + @pytest.mark.parametrize("async_scheduling", [True, False]) + def test_full_attn_store_excludes_trailing_block( + self, request_runner, async_scheduling: bool + ): + """Eagle full-attention group stores all blocks except the trailing + one. + + Setup: 2 groups — group 0 is normal full-attention, group 1 is + eagle full-attention. With a 3-block prompt, group 1 should store + only blocks 0 and 1, skipping block 2 (the volatile tail). + """ + block_size = 4 + block_size_factor = 1 + offloaded_block_size = block_size * block_size_factor + num_gpu_blocks = 100 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=num_gpu_blocks, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + block_size_factor=block_size_factor, + ) + + kv_group_configs = runner.connector_scheduler.config.kv_group_configs + assert len(kv_group_configs) == 2 + assert not kv_group_configs[0].is_eagle_group + assert kv_group_configs[1].is_eagle_group + + runner.new_request(token_ids=[0] * offloaded_block_size * 3) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=( + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + ), + expected_flushed=( + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + ) + if not async_scheduling + else (), + ) + + @pytest.mark.parametrize("async_scheduling", [True, False]) + def test_sw_store_excludes_trailing_block( + self, request_runner, async_scheduling: bool + ): + """Eagle sliding-window group stores all blocks except the trailing + one.""" + block_size = 4 + sliding_window = 8 + num_gpu_blocks = 100 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sliding_window, + ), + is_eagle_group=True, + ), + ] + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=num_gpu_blocks, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + ) + + kv_group_configs = runner.connector_scheduler.config.kv_group_configs + assert len(kv_group_configs) == 1 + assert kv_group_configs[0].is_eagle_group + assert kv_group_configs[0].sliding_window_size_in_blocks == 2 + + runner.new_request(token_ids=[0] * block_size * 3) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=((0, 0), (0, 1)), + expected_flushed=((0, 0), (0, 1)) if not async_scheduling else (), + ) + + @pytest.mark.parametrize("async_scheduling", [True, False]) + def test_single_block_nothing_stored(self, request_runner, async_scheduling: bool): + """An eagle group with only one block stores nothing: that block is + the tail.""" + block_size = 4 + block_size_factor = 1 + offloaded_block_size = block_size * block_size_factor + num_gpu_blocks = 100 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=num_gpu_blocks, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + block_size_factor=block_size_factor, + ) + + runner.new_request(token_ids=[0] * offloaded_block_size) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=()) + runner.manager.prepare_store.assert_not_called() + + @pytest.mark.parametrize("async_scheduling", [True, False]) + def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool): + """Eagle group constrains load: convergence tightens both groups. + + Store 3 offloaded blocks per group (eagle group skips tail → stores + 2). Then a new request loads from CPU. The eagle group's post-pop hit + (2) does not tighten below group 0's hit (3), so both groups load + normally. + """ + block_size = 4 + block_size_factor = 1 + offloaded_block_size = block_size * block_size_factor + num_gpu_blocks = 100 + + kv_cache_groups = [ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=1, + dtype=torch.float32, + ), + is_eagle_group=True, + ), + ] + + runner = request_runner( + block_size=block_size, + num_gpu_blocks=num_gpu_blocks, + async_scheduling=async_scheduling, + kv_cache_groups=kv_cache_groups, + block_size_factor=block_size_factor, + ) + + runner.new_request(token_ids=[0] * offloaded_block_size * 3) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(keys) + ) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_stored=( + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + ), + expected_flushed=( + (0, 0), + (0, 1), + (0, 2), + (1, 0), + (1, 1), + ) + if not async_scheduling + else (), + ) + + runner.scheduler.reset_prefix_cache() + + runner.new_request(token_ids=[0] * offloaded_block_size * 3 + [1]) + runner.manager.lookup.return_value = True + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output([]) + ) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + expected_loaded=( + (0, 0), + (0, 1), + (1, 0), + (1, 1), + ), + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 1d3d83709b..443d5b28d5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -77,6 +77,10 @@ class GroupOffloadConfig(NamedTuple): # than the MLA full-attention group). # None for full-attention groups or when the optimization doesn't apply. alignment_block_count: int | None = None + # True for EAGLE/MTP draft-model attention groups. The trailing block + # of these groups is volatile and lacks a stable hash, so it must + # be excluded from store and load scheduling. + is_eagle_group: bool = False def get_sliding_window_size_in_blocks( @@ -155,6 +159,27 @@ class SchedulerOffloadConfig(NamedTuple): return None return per_segment + eagle_groups = { + idx + for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups) + if g.is_eagle_group + } + + use_eagle = ( + spec.vllm_config.speculative_config is not None + and spec.vllm_config.speculative_config.use_eagle() + ) + if use_eagle and not eagle_groups: + eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups))) + + if eagle_groups: + logger.info( + "KV offloading: EAGLE/MTP draft attention groups %s " + "detected. The trailing block of these groups will be " + "excluded from offloading due to volatility.", + sorted(eagle_groups), + ) + return cls( num_workers=spec.vllm_config.parallel_config.world_size, kv_group_configs=tuple( @@ -175,6 +200,7 @@ class SchedulerOffloadConfig(NamedTuple): alignment_block_count=_alignment_block_count( gpu_block_size * spec.block_size_factor, sw ), + is_eagle_group=idx in eagle_groups, ) for idx, gpu_block_size in enumerate(spec.gpu_block_size) ), @@ -436,6 +462,11 @@ class OffloadingConnectorScheduler: num_hit_tokens: int = 0 defer_lookup = False lookup_groups = self._lookup_groups + + # Tracks which eagle groups have already popped their volatile trailing block + # in the current convergence iteration. Reset when a non-eagle group + # tightens the hit boundary, requiring a fresh pop. + eagle_verified: set[int] = set() while lookup_groups: looked_up_sliding_window: bool = False groups_iter = iter(lookup_groups) @@ -453,6 +484,10 @@ class OffloadingConnectorScheduler: >= req_status.req.num_tokens // offloaded_block_size ) + is_eagle_unverified = ( + group_config.is_eagle_group and group_idx not in eagle_verified + ) + # Constrain to block-aligned boundary for this group max_hit_size_tokens = min( max_hit_size_tokens, len(offload_keys) * offloaded_block_size @@ -461,15 +496,25 @@ class OffloadingConnectorScheduler: # we can only load less than a block, better skip return 0 - num_blocks = min( - cdiv(max_hit_size_tokens, offloaded_block_size), len(offload_keys) - ) - start_block_idx = num_computed_tokens // offloaded_block_size - offload_keys = offload_keys[start_block_idx:num_blocks] sliding_window_size_in_blocks = ( group_config.sliding_window_size_in_blocks ) + # For eagle groups, query one extra block that will be popped. + # We only need to increase the query size for sliding window groups. + query_max = max_hit_size_tokens + if is_eagle_unverified and sliding_window_size_in_blocks is not None: + query_max = min( + max_hit_size_tokens + offloaded_block_size, + len(offload_keys) * offloaded_block_size, + ) + + num_blocks = min( + cdiv(query_max, offloaded_block_size), len(offload_keys) + ) + start_block_idx = num_computed_tokens // offloaded_block_size + offload_keys = offload_keys[start_block_idx:num_blocks] + # end index (in the sliced offload_keys) up to which we # have backend-confirmed hits num_hit_blocks: int | None @@ -478,9 +523,12 @@ class OffloadingConnectorScheduler: offload_keys, req_status.req_context ) else: + required_window = sliding_window_size_in_blocks + if is_eagle_unverified: + required_window += 1 num_hit_blocks = self._sliding_window_lookup( offload_keys, - sliding_window_size_in_blocks, + required_window, req_status.req_context, ) if num_hit_blocks == 0: @@ -489,6 +537,10 @@ class OffloadingConnectorScheduler: if num_hit_blocks is None: defer_lookup = True else: + if is_eagle_unverified: + num_hit_blocks -= 1 + eagle_verified.add(group_idx) + max_hit_size_tokens = min( max_hit_size_tokens, offloaded_block_size * (start_block_idx + num_hit_blocks), @@ -500,6 +552,8 @@ class OffloadingConnectorScheduler: return 0 if new_num_hit_tokens < num_hit_tokens: + if not group_config.is_eagle_group: + eagle_verified.clear() if defer_lookup: # make another iteration on all groups to check # if we still need to defer lookup @@ -791,6 +845,9 @@ class OffloadingConnectorScheduler: self.config.kv_group_configs, req_status.group_states ): num_blocks = num_offloadable_tokens // group_config.offloaded_block_size + if group_config.is_eagle_group: + num_blocks = max(0, num_blocks - 1) + start_block_idx = group_state.next_stored_block_idx if num_blocks <= start_block_idx: continue