[Bugfix] Prevent cuMemcpyBatchAsync segfault with MTP and KV offloading (#44784)

Signed-off-by: joshua <joshua.abraham@multicorewareinc.com>
Co-authored-by: joshua <joshua.abraham@multicorewareinc.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
joshua abraham
2026-06-16 07:58:39 +00:00
committed by GitHub
co-authored by joshua Or Ozeri
parent a7fdfeef72
commit 7ad894c86a
2 changed files with 745 additions and 6 deletions
@@ -14,6 +14,7 @@ from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
from vllm.distributed.kv_events import BlockRemoved, BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
OffloadingConnectorScheduler,
RequestOffloadState,
)
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
@@ -28,6 +29,7 @@ from vllm.v1.kv_offload.base import (
ReqContext,
RequestOffloadingContext,
get_offload_block_hash,
make_offload_key,
)
from vllm.v1.request import RequestStatus
@@ -1432,3 +1434,683 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
# The external lookup must have been completely skipped.
runner.manager.lookup.assert_not_called()
# ---------------------------------------------------------------------------
# Eagle/MTP test class
# ---------------------------------------------------------------------------
class TestEagle:
"""Tests for Eagle/MTP speculative decoding support in the offloading
connector scheduler — both _lookup() unit tests and integration tests."""
# -------------------------------------------------------------------
# Helpers
# -------------------------------------------------------------------
@staticmethod
def _group_keys(group_idx: int, int_hashes: list[int]) -> list:
return [make_offload_key(str(h).encode(), group_idx) for h in int_hashes]
@staticmethod
def _make_req_status(
scheduler: OffloadingConnectorScheduler,
*,
num_tokens: int,
num_computed_tokens: int = 0,
offload_keys_per_group: list[list[int]],
) -> RequestOffloadState:
"""Build RequestOffloadState with synthetic offload keys."""
req = MagicMock()
req.request_id = "test-req"
req.num_tokens = num_tokens
req.kv_transfer_params = None
state = RequestOffloadState(
config=scheduler.config,
req=req,
req_context=ReqContext(req_id="test-req"),
offloading_context=RequestOffloadingContext(
policy=OffloadPolicy.BLOCK_LEVEL
),
num_locally_computed_tokens=num_computed_tokens,
)
for idx, (gs, hashes) in enumerate(
zip(state.group_states, offload_keys_per_group)
):
gs.offload_keys = TestEagle._group_keys(
scheduler.config.kv_group_configs[idx].group_idx, hashes
)
return state
# -------------------------------------------------------------------
# Lookup unit tests: call _lookup() directly via request_runner
# -------------------------------------------------------------------
def test_full_attn_lookup_pops_one_block(self, request_runner):
"""Full-attn eagle group with 3 blocks all hit → pop to 2 blocks."""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched, num_tokens=12, offload_keys_per_group=[[1, 2, 3]]
)
# 3 hits, pop to 2 → 2 * block_size = 8 tokens loadable
assert sched._lookup(req_status) == 8
def test_full_attn_lookup_single_block_returns_zero(self, request_runner):
"""Full-attn eagle group with 1 block hit → pop to 0 → returns 0."""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched, num_tokens=4, offload_keys_per_group=[[1]]
)
# 1 hit, pop to 0 → new_num_hit_tokens < block_size → return 0
assert sched._lookup(req_status) == 0
def test_full_attn_lookup_no_hits_returns_zero(self, request_runner):
"""Full-attn eagle group with 0 hits returns 0 before pop."""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.return_value = False
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched, num_tokens=8, offload_keys_per_group=[[1, 2]]
)
assert sched._lookup(req_status) == 0
def test_sw_lookup_inflates_query_max(self, request_runner):
"""SW eagle group inflates query_max so _sliding_window_lookup gets
one extra key beyond what max_hit_size_tokens alone would yield.
With block_size=4, W=2, eagle, num_tokens=13, 4 keys all hitting:
- max_hit = 13-1 = 12 (SW reduction)
- Without inflation: num_blocks = cdiv(12,4) = 3 → only 3 keys
- With inflation: query_max = min(12+4, 4*4=16) = 16,
num_blocks = cdiv(16,4) = 4 → 4 keys passed to SW
- SW finds window of 3 (required=W+1=3) at idx 1 → returns 4
- Pop: 4-1=3 → max_hit = min(12, 12) = 12. Result: 12.
"""
block_size = 4
sw_blocks = 2
groups = [
KVCacheGroupSpec(
["layer0"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sw_blocks * block_size,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2, 3, 4}
)
sched = runner.connector_scheduler
captured_keys: list = []
orig_sw_lookup = type(sched)._sliding_window_lookup
def capturing_sw_lookup(self_arg, keys, window, req_context):
captured_keys.append(list(keys))
return orig_sw_lookup(self_arg, keys, window, req_context)
sched._sliding_window_lookup = lambda keys, window, req_ctx: (
capturing_sw_lookup(sched, keys, window, req_ctx)
)
req_status = self._make_req_status(
sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3, 4]]
)
result = sched._lookup(req_status)
assert len(captured_keys) == 1
# Inflation bumped from 3 keys (cdiv(12,4)) to 4 keys (cdiv(16,4))
assert len(captured_keys[0]) == 4
# SW finds window of 3 → returns 4, pop to 3 → 3*4=12
assert result == 12
def test_sw_lookup_requires_extra_window_block(self, request_runner):
"""SW eagle with W=2 and only 2 keys (both hit) uses prefix fallback.
Since required_window = W+1 = 3 but only 2 keys are available
(inflation is capped by len(offload_keys)), _sliding_window_lookup
can never find a window of 3. It falls back to prefix count (2).
Pop: 2-1=1 → max_hit = 4. Result: 4 tokens (degraded from full hit).
"""
block_size = 4
sw_blocks = 2
groups = [
KVCacheGroupSpec(
["layer0"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sw_blocks * block_size,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched, num_tokens=9, offload_keys_per_group=[[1, 2]]
)
# Prefix fallback returns 2, pop to 1 → 1*4 = 4 tokens
assert sched._lookup(req_status) == 4
def test_sw_lookup_w_plus_one_hits_returns_w_blocks(self, request_runner):
"""SW eagle with W=2, 3 contiguous hits → pop to 2 → returns 2*bs."""
block_size = 4
sw_blocks = 2
groups = [
KVCacheGroupSpec(
["layer0"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sw_blocks * block_size,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
)
sched = runner.connector_scheduler
# num_tokens=13 → max_hit=13-1=12, query_max=min(12+4,12)=12
# num_blocks=cdiv(12,4)=3, keys=[1,2,3], required_window=3
# SW finds window of 3, pop to 2 → 2*4=8
req_status = self._make_req_status(
sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3]]
)
assert sched._lookup(req_status) == 8
def test_eagle_verified_prevents_double_pop(self, request_runner):
"""Once an eagle group has popped, it doesn't pop again on re-iteration.
Setup: group 0 = non-eagle full-attn (3 blocks), group 1 = eagle
full-attn (3 blocks). Both see all hits. Eagle pops to 2 and tightens
max_hit to 8. Group 0 re-runs (convergence) but since eagle_verified
contains group 1, it won't pop again — result stays at 8 tokens.
"""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=False,
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched,
num_tokens=12,
offload_keys_per_group=[[1, 2, 3], [1, 2, 3]],
)
# Group 0: prefix finds 3 → max_hit=12, num_hit=12
# Group 1 (eagle): prefix finds 3, pop to 2 → max_hit=8, num_hit=8
# num_hit(8) < prev num_hit(12) AND group IS eagle → no clear
# No re-iteration triggered (eagle shrink doesn't trigger re-loop)
# Final: 8 tokens
assert sched._lookup(req_status) == 8
def test_non_eagle_tighten_clears_eagle_verified(self, request_runner):
"""Non-eagle group tightening clears eagle_verified → eagle re-pops.
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4.
This clears eagle_verified. Group 1 runs with max_hit=4 → only 1
key queried, 1 hit, pop to 0 → returns 0.
"""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=False,
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
# Group 0 keys [10,11,12]: only 10 hits.
# Group 1 keys [1,2,3]: all hit.
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {10, 1, 2, 3}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched,
num_tokens=12,
offload_keys_per_group=[[10, 11, 12], [1, 2, 3]],
)
# Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4
# Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1].
# Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0
assert sched._lookup(req_status) == 0
def test_eagle_verified_survives_eagle_tighten(self, request_runner):
"""Eagle group tightening does NOT clear eagle_verified.
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2
(max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared.
Result: 8 tokens (eagle only pops once).
"""
block_size = 4
groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=False,
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
runner.manager.lookup.side_effect = lambda key, req_context: (
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
)
sched = runner.connector_scheduler
req_status = self._make_req_status(
sched,
num_tokens=12,
offload_keys_per_group=[[1, 2, 3], [1, 2, 3]],
)
# Group 0: 3 hits → max_hit=12, num_hit=12
# Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8
# Tightened but IS eagle → no clear. No re-iteration.
assert sched._lookup(req_status) == 8
# -------------------------------------------------------------------
# Integration tests: store and load via request_runner
# -------------------------------------------------------------------
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_full_attn_store_excludes_trailing_block(
self, request_runner, async_scheduling: bool
):
"""Eagle full-attention group stores all blocks except the trailing
one.
Setup: 2 groups — group 0 is normal full-attention, group 1 is
eagle full-attention. With a 3-block prompt, group 1 should store
only blocks 0 and 1, skipping block 2 (the volatile tail).
"""
block_size = 4
block_size_factor = 1
offloaded_block_size = block_size * block_size_factor
num_gpu_blocks = 100
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
block_size_factor=block_size_factor,
)
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
assert len(kv_group_configs) == 2
assert not kv_group_configs[0].is_eagle_group
assert kv_group_configs[1].is_eagle_group
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(
(0, 0),
(0, 1),
(0, 2),
(1, 0),
(1, 1),
),
expected_flushed=(
(0, 0),
(0, 1),
(0, 2),
(1, 0),
(1, 1),
)
if not async_scheduling
else (),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_sw_store_excludes_trailing_block(
self, request_runner, async_scheduling: bool
):
"""Eagle sliding-window group stores all blocks except the trailing
one."""
block_size = 4
sliding_window = 8
num_gpu_blocks = 100
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
)
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
assert len(kv_group_configs) == 1
assert kv_group_configs[0].is_eagle_group
assert kv_group_configs[0].sliding_window_size_in_blocks == 2
runner.new_request(token_ids=[0] * block_size * 3)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=((0, 0), (0, 1)),
expected_flushed=((0, 0), (0, 1)) if not async_scheduling else (),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_single_block_nothing_stored(self, request_runner, async_scheduling: bool):
"""An eagle group with only one block stores nothing: that block is
the tail."""
block_size = 4
block_size_factor = 1
offloaded_block_size = block_size * block_size_factor
num_gpu_blocks = 100
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
block_size_factor=block_size_factor,
)
runner.new_request(token_ids=[0] * offloaded_block_size)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=())
runner.manager.prepare_store.assert_not_called()
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool):
"""Eagle group constrains load: convergence tightens both groups.
Store 3 offloaded blocks per group (eagle group skips tail → stores
2). Then a new request loads from CPU. The eagle group's post-pop hit
(2) does not tighten below group 0's hit (3), so both groups load
normally.
"""
block_size = 4
block_size_factor = 1
offloaded_block_size = block_size * block_size_factor
num_gpu_blocks = 100
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=2,
head_size=1,
dtype=torch.float32,
),
is_eagle_group=True,
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
block_size_factor=block_size_factor,
)
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(
(0, 0),
(0, 1),
(0, 2),
(1, 0),
(1, 1),
),
expected_flushed=(
(0, 0),
(0, 1),
(0, 2),
(1, 0),
(1, 1),
)
if not async_scheduling
else (),
)
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * offloaded_block_size * 3 + [1])
runner.manager.lookup.return_value = True
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output([])
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_loaded=(
(0, 0),
(0, 1),
(1, 0),
(1, 1),
),
)
@@ -77,6 +77,10 @@ class GroupOffloadConfig(NamedTuple):
# than the MLA full-attention group).
# None for full-attention groups or when the optimization doesn't apply.
alignment_block_count: int | None = None
# True for EAGLE/MTP draft-model attention groups. The trailing block
# of these groups is volatile and lacks a stable hash, so it must
# be excluded from store and load scheduling.
is_eagle_group: bool = False
def get_sliding_window_size_in_blocks(
@@ -155,6 +159,27 @@ class SchedulerOffloadConfig(NamedTuple):
return None
return per_segment
eagle_groups = {
idx
for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups)
if g.is_eagle_group
}
use_eagle = (
spec.vllm_config.speculative_config is not None
and spec.vllm_config.speculative_config.use_eagle()
)
if use_eagle and not eagle_groups:
eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups)))
if eagle_groups:
logger.info(
"KV offloading: EAGLE/MTP draft attention groups %s "
"detected. The trailing block of these groups will be "
"excluded from offloading due to volatility.",
sorted(eagle_groups),
)
return cls(
num_workers=spec.vllm_config.parallel_config.world_size,
kv_group_configs=tuple(
@@ -175,6 +200,7 @@ class SchedulerOffloadConfig(NamedTuple):
alignment_block_count=_alignment_block_count(
gpu_block_size * spec.block_size_factor, sw
),
is_eagle_group=idx in eagle_groups,
)
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
),
@@ -436,6 +462,11 @@ class OffloadingConnectorScheduler:
num_hit_tokens: int = 0
defer_lookup = False
lookup_groups = self._lookup_groups
# Tracks which eagle groups have already popped their volatile trailing block
# in the current convergence iteration. Reset when a non-eagle group
# tightens the hit boundary, requiring a fresh pop.
eagle_verified: set[int] = set()
while lookup_groups:
looked_up_sliding_window: bool = False
groups_iter = iter(lookup_groups)
@@ -453,6 +484,10 @@ class OffloadingConnectorScheduler:
>= req_status.req.num_tokens // offloaded_block_size
)
is_eagle_unverified = (
group_config.is_eagle_group and group_idx not in eagle_verified
)
# Constrain to block-aligned boundary for this group
max_hit_size_tokens = min(
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
@@ -461,15 +496,25 @@ class OffloadingConnectorScheduler:
# we can only load less than a block, better skip
return 0
num_blocks = min(
cdiv(max_hit_size_tokens, offloaded_block_size), len(offload_keys)
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
sliding_window_size_in_blocks = (
group_config.sliding_window_size_in_blocks
)
# For eagle groups, query one extra block that will be popped.
# We only need to increase the query size for sliding window groups.
query_max = max_hit_size_tokens
if is_eagle_unverified and sliding_window_size_in_blocks is not None:
query_max = min(
max_hit_size_tokens + offloaded_block_size,
len(offload_keys) * offloaded_block_size,
)
num_blocks = min(
cdiv(query_max, offloaded_block_size), len(offload_keys)
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
# end index (in the sliced offload_keys) up to which we
# have backend-confirmed hits
num_hit_blocks: int | None
@@ -478,9 +523,12 @@ class OffloadingConnectorScheduler:
offload_keys, req_status.req_context
)
else:
required_window = sliding_window_size_in_blocks
if is_eagle_unverified:
required_window += 1
num_hit_blocks = self._sliding_window_lookup(
offload_keys,
sliding_window_size_in_blocks,
required_window,
req_status.req_context,
)
if num_hit_blocks == 0:
@@ -489,6 +537,10 @@ class OffloadingConnectorScheduler:
if num_hit_blocks is None:
defer_lookup = True
else:
if is_eagle_unverified:
num_hit_blocks -= 1
eagle_verified.add(group_idx)
max_hit_size_tokens = min(
max_hit_size_tokens,
offloaded_block_size * (start_block_idx + num_hit_blocks),
@@ -500,6 +552,8 @@ class OffloadingConnectorScheduler:
return 0
if new_num_hit_tokens < num_hit_tokens:
if not group_config.is_eagle_group:
eagle_verified.clear()
if defer_lookup:
# make another iteration on all groups to check
# if we still need to defer lookup
@@ -791,6 +845,9 @@ class OffloadingConnectorScheduler:
self.config.kv_group_configs, req_status.group_states
):
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
if group_config.is_eagle_group:
num_blocks = max(0, num_blocks - 1)
start_block_idx = group_state.next_stored_block_idx
if num_blocks <= start_block_idx:
continue