[Bugfix] Prevent cuMemcpyBatchAsync segfault with MTP and KV offloading (#44784)
Signed-off-by: joshua <joshua.abraham@multicorewareinc.com> Co-authored-by: joshua <joshua.abraham@multicorewareinc.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
joshua
Or Ozeri
parent
a7fdfeef72
commit
7ad894c86a
@@ -14,6 +14,7 @@ from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.distributed.kv_events import BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -28,6 +29,7 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
get_offload_block_hash,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
@@ -1432,3 +1434,683 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
|
||||
# The external lookup must have been completely skipped.
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Eagle/MTP test class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEagle:
|
||||
"""Tests for Eagle/MTP speculative decoding support in the offloading
|
||||
connector scheduler — both _lookup() unit tests and integration tests."""
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Helpers
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _group_keys(group_idx: int, int_hashes: list[int]) -> list:
|
||||
return [make_offload_key(str(h).encode(), group_idx) for h in int_hashes]
|
||||
|
||||
@staticmethod
|
||||
def _make_req_status(
|
||||
scheduler: OffloadingConnectorScheduler,
|
||||
*,
|
||||
num_tokens: int,
|
||||
num_computed_tokens: int = 0,
|
||||
offload_keys_per_group: list[list[int]],
|
||||
) -> RequestOffloadState:
|
||||
"""Build RequestOffloadState with synthetic offload keys."""
|
||||
req = MagicMock()
|
||||
req.request_id = "test-req"
|
||||
req.num_tokens = num_tokens
|
||||
req.kv_transfer_params = None
|
||||
|
||||
state = RequestOffloadState(
|
||||
config=scheduler.config,
|
||||
req=req,
|
||||
req_context=ReqContext(req_id="test-req"),
|
||||
offloading_context=RequestOffloadingContext(
|
||||
policy=OffloadPolicy.BLOCK_LEVEL
|
||||
),
|
||||
num_locally_computed_tokens=num_computed_tokens,
|
||||
)
|
||||
for idx, (gs, hashes) in enumerate(
|
||||
zip(state.group_states, offload_keys_per_group)
|
||||
):
|
||||
gs.offload_keys = TestEagle._group_keys(
|
||||
scheduler.config.kv_group_configs[idx].group_idx, hashes
|
||||
)
|
||||
return state
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Lookup unit tests: call _lookup() directly via request_runner
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def test_full_attn_lookup_pops_one_block(self, request_runner):
|
||||
"""Full-attn eagle group with 3 blocks all hit → pop to 2 blocks."""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=12, offload_keys_per_group=[[1, 2, 3]]
|
||||
)
|
||||
# 3 hits, pop to 2 → 2 * block_size = 8 tokens loadable
|
||||
assert sched._lookup(req_status) == 8
|
||||
|
||||
def test_full_attn_lookup_single_block_returns_zero(self, request_runner):
|
||||
"""Full-attn eagle group with 1 block hit → pop to 0 → returns 0."""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=4, offload_keys_per_group=[[1]]
|
||||
)
|
||||
# 1 hit, pop to 0 → new_num_hit_tokens < block_size → return 0
|
||||
assert sched._lookup(req_status) == 0
|
||||
|
||||
def test_full_attn_lookup_no_hits_returns_zero(self, request_runner):
|
||||
"""Full-attn eagle group with 0 hits returns 0 before pop."""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.return_value = False
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=8, offload_keys_per_group=[[1, 2]]
|
||||
)
|
||||
assert sched._lookup(req_status) == 0
|
||||
|
||||
def test_sw_lookup_inflates_query_max(self, request_runner):
|
||||
"""SW eagle group inflates query_max so _sliding_window_lookup gets
|
||||
one extra key beyond what max_hit_size_tokens alone would yield.
|
||||
|
||||
With block_size=4, W=2, eagle, num_tokens=13, 4 keys all hitting:
|
||||
- max_hit = 13-1 = 12 (SW reduction)
|
||||
- Without inflation: num_blocks = cdiv(12,4) = 3 → only 3 keys
|
||||
- With inflation: query_max = min(12+4, 4*4=16) = 16,
|
||||
num_blocks = cdiv(16,4) = 4 → 4 keys passed to SW
|
||||
- SW finds window of 3 (required=W+1=3) at idx 1 → returns 4
|
||||
- Pop: 4-1=3 → max_hit = min(12, 12) = 12. Result: 12.
|
||||
"""
|
||||
block_size = 4
|
||||
sw_blocks = 2
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sw_blocks * block_size,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2, 3, 4}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
|
||||
captured_keys: list = []
|
||||
orig_sw_lookup = type(sched)._sliding_window_lookup
|
||||
|
||||
def capturing_sw_lookup(self_arg, keys, window, req_context):
|
||||
captured_keys.append(list(keys))
|
||||
return orig_sw_lookup(self_arg, keys, window, req_context)
|
||||
|
||||
sched._sliding_window_lookup = lambda keys, window, req_ctx: (
|
||||
capturing_sw_lookup(sched, keys, window, req_ctx)
|
||||
)
|
||||
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3, 4]]
|
||||
)
|
||||
result = sched._lookup(req_status)
|
||||
assert len(captured_keys) == 1
|
||||
# Inflation bumped from 3 keys (cdiv(12,4)) to 4 keys (cdiv(16,4))
|
||||
assert len(captured_keys[0]) == 4
|
||||
# SW finds window of 3 → returns 4, pop to 3 → 3*4=12
|
||||
assert result == 12
|
||||
|
||||
def test_sw_lookup_requires_extra_window_block(self, request_runner):
|
||||
"""SW eagle with W=2 and only 2 keys (both hit) uses prefix fallback.
|
||||
|
||||
Since required_window = W+1 = 3 but only 2 keys are available
|
||||
(inflation is capped by len(offload_keys)), _sliding_window_lookup
|
||||
can never find a window of 3. It falls back to prefix count (2).
|
||||
Pop: 2-1=1 → max_hit = 4. Result: 4 tokens (degraded from full hit).
|
||||
"""
|
||||
block_size = 4
|
||||
sw_blocks = 2
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sw_blocks * block_size,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=9, offload_keys_per_group=[[1, 2]]
|
||||
)
|
||||
# Prefix fallback returns 2, pop to 1 → 1*4 = 4 tokens
|
||||
assert sched._lookup(req_status) == 4
|
||||
|
||||
def test_sw_lookup_w_plus_one_hits_returns_w_blocks(self, request_runner):
|
||||
"""SW eagle with W=2, 3 contiguous hits → pop to 2 → returns 2*bs."""
|
||||
block_size = 4
|
||||
sw_blocks = 2
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sw_blocks * block_size,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
# num_tokens=13 → max_hit=13-1=12, query_max=min(12+4,12)=12
|
||||
# num_blocks=cdiv(12,4)=3, keys=[1,2,3], required_window=3
|
||||
# SW finds window of 3, pop to 2 → 2*4=8
|
||||
req_status = self._make_req_status(
|
||||
sched, num_tokens=13, offload_keys_per_group=[[1, 2, 3]]
|
||||
)
|
||||
assert sched._lookup(req_status) == 8
|
||||
|
||||
def test_eagle_verified_prevents_double_pop(self, request_runner):
|
||||
"""Once an eagle group has popped, it doesn't pop again on re-iteration.
|
||||
|
||||
Setup: group 0 = non-eagle full-attn (3 blocks), group 1 = eagle
|
||||
full-attn (3 blocks). Both see all hits. Eagle pops to 2 and tightens
|
||||
max_hit to 8. Group 0 re-runs (convergence) but since eagle_verified
|
||||
contains group 1, it won't pop again — result stays at 8 tokens.
|
||||
"""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=False,
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched,
|
||||
num_tokens=12,
|
||||
offload_keys_per_group=[[1, 2, 3], [1, 2, 3]],
|
||||
)
|
||||
# Group 0: prefix finds 3 → max_hit=12, num_hit=12
|
||||
# Group 1 (eagle): prefix finds 3, pop to 2 → max_hit=8, num_hit=8
|
||||
# num_hit(8) < prev num_hit(12) AND group IS eagle → no clear
|
||||
# No re-iteration triggered (eagle shrink doesn't trigger re-loop)
|
||||
# Final: 8 tokens
|
||||
assert sched._lookup(req_status) == 8
|
||||
|
||||
def test_non_eagle_tighten_clears_eagle_verified(self, request_runner):
|
||||
"""Non-eagle group tightening clears eagle_verified → eagle re-pops.
|
||||
|
||||
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
|
||||
Group 0 has only 1 hit (out of 3 keys) → max_hit tightens to 4.
|
||||
This clears eagle_verified. Group 1 runs with max_hit=4 → only 1
|
||||
key queried, 1 hit, pop to 0 → returns 0.
|
||||
"""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=False,
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
# Group 0 keys [10,11,12]: only 10 hits.
|
||||
# Group 1 keys [1,2,3]: all hit.
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {10, 1, 2, 3}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched,
|
||||
num_tokens=12,
|
||||
offload_keys_per_group=[[10, 11, 12], [1, 2, 3]],
|
||||
)
|
||||
# Group 0 (non-eagle FA): prefix finds 1 hit → max_hit=4, num_hit=4
|
||||
# Group 1 (eagle FA): max_hit=4 → num_blocks=1, keys=[1].
|
||||
# Finds 1 hit, pop to 0 → new_num_hit = 0 < block_size → return 0
|
||||
assert sched._lookup(req_status) == 0
|
||||
|
||||
def test_eagle_verified_survives_eagle_tighten(self, request_runner):
|
||||
"""Eagle group tightening does NOT clear eagle_verified.
|
||||
|
||||
Groups: 0=non-eagle full-attn, 1=eagle full-attn.
|
||||
Group 0 finds 3 hits (max_hit=12). Group 1 finds 3 hits, pops to 2
|
||||
(max_hit=8). Since group 1 IS eagle, eagle_verified is NOT cleared.
|
||||
Result: 8 tokens (eagle only pops once).
|
||||
"""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=False,
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
runner.manager.lookup.side_effect = lambda key, req_context: (
|
||||
int(get_offload_block_hash(key).decode()) in {1, 2, 3}
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
req_status = self._make_req_status(
|
||||
sched,
|
||||
num_tokens=12,
|
||||
offload_keys_per_group=[[1, 2, 3], [1, 2, 3]],
|
||||
)
|
||||
# Group 0: 3 hits → max_hit=12, num_hit=12
|
||||
# Group 1 (eagle): 3 hits, pop to 2 → max_hit=8, num_hit=8
|
||||
# Tightened but IS eagle → no clear. No re-iteration.
|
||||
assert sched._lookup(req_status) == 8
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Integration tests: store and load via request_runner
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_full_attn_store_excludes_trailing_block(
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle full-attention group stores all blocks except the trailing
|
||||
one.
|
||||
|
||||
Setup: 2 groups — group 0 is normal full-attention, group 1 is
|
||||
eagle full-attention. With a 3-block prompt, group 1 should store
|
||||
only blocks 0 and 1, skipping block 2 (the volatile tail).
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
)
|
||||
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 2
|
||||
assert not kv_group_configs[0].is_eagle_group
|
||||
assert kv_group_configs[1].is_eagle_group
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(0, 2),
|
||||
(1, 0),
|
||||
(1, 1),
|
||||
),
|
||||
expected_flushed=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(0, 2),
|
||||
(1, 0),
|
||||
(1, 1),
|
||||
)
|
||||
if not async_scheduling
|
||||
else (),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_sw_store_excludes_trailing_block(
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle sliding-window group stores all blocks except the trailing
|
||||
one."""
|
||||
block_size = 4
|
||||
sliding_window = 8
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 1
|
||||
assert kv_group_configs[0].is_eagle_group
|
||||
assert kv_group_configs[0].sliding_window_size_in_blocks == 2
|
||||
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=((0, 0), (0, 1)),
|
||||
expected_flushed=((0, 0), (0, 1)) if not async_scheduling else (),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_single_block_nothing_stored(self, request_runner, async_scheduling: bool):
|
||||
"""An eagle group with only one block stores nothing: that block is
|
||||
the tail."""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_stored=())
|
||||
runner.manager.prepare_store.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_full_attn_store_then_load(self, request_runner, async_scheduling: bool):
|
||||
"""Eagle group constrains load: convergence tightens both groups.
|
||||
|
||||
Store 3 offloaded blocks per group (eagle group skips tail → stores
|
||||
2). Then a new request loads from CPU. The eagle group's post-pop hit
|
||||
(2) does not tighten below group 0's hit (3), so both groups load
|
||||
normally.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 1
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
num_gpu_blocks = 100
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
is_eagle_group=True,
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
block_size_factor=block_size_factor,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(0, 2),
|
||||
(1, 0),
|
||||
(1, 1),
|
||||
),
|
||||
expected_flushed=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(0, 2),
|
||||
(1, 0),
|
||||
(1, 1),
|
||||
)
|
||||
if not async_scheduling
|
||||
else (),
|
||||
)
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 3 + [1])
|
||||
runner.manager.lookup.return_value = True
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_loaded=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(1, 0),
|
||||
(1, 1),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -77,6 +77,10 @@ class GroupOffloadConfig(NamedTuple):
|
||||
# than the MLA full-attention group).
|
||||
# None for full-attention groups or when the optimization doesn't apply.
|
||||
alignment_block_count: int | None = None
|
||||
# True for EAGLE/MTP draft-model attention groups. The trailing block
|
||||
# of these groups is volatile and lacks a stable hash, so it must
|
||||
# be excluded from store and load scheduling.
|
||||
is_eagle_group: bool = False
|
||||
|
||||
|
||||
def get_sliding_window_size_in_blocks(
|
||||
@@ -155,6 +159,27 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
return None
|
||||
return per_segment
|
||||
|
||||
eagle_groups = {
|
||||
idx
|
||||
for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups)
|
||||
if g.is_eagle_group
|
||||
}
|
||||
|
||||
use_eagle = (
|
||||
spec.vllm_config.speculative_config is not None
|
||||
and spec.vllm_config.speculative_config.use_eagle()
|
||||
)
|
||||
if use_eagle and not eagle_groups:
|
||||
eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups)))
|
||||
|
||||
if eagle_groups:
|
||||
logger.info(
|
||||
"KV offloading: EAGLE/MTP draft attention groups %s "
|
||||
"detected. The trailing block of these groups will be "
|
||||
"excluded from offloading due to volatility.",
|
||||
sorted(eagle_groups),
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=spec.vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=tuple(
|
||||
@@ -175,6 +200,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
alignment_block_count=_alignment_block_count(
|
||||
gpu_block_size * spec.block_size_factor, sw
|
||||
),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
)
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
|
||||
),
|
||||
@@ -436,6 +462,11 @@ class OffloadingConnectorScheduler:
|
||||
num_hit_tokens: int = 0
|
||||
defer_lookup = False
|
||||
lookup_groups = self._lookup_groups
|
||||
|
||||
# Tracks which eagle groups have already popped their volatile trailing block
|
||||
# in the current convergence iteration. Reset when a non-eagle group
|
||||
# tightens the hit boundary, requiring a fresh pop.
|
||||
eagle_verified: set[int] = set()
|
||||
while lookup_groups:
|
||||
looked_up_sliding_window: bool = False
|
||||
groups_iter = iter(lookup_groups)
|
||||
@@ -453,6 +484,10 @@ class OffloadingConnectorScheduler:
|
||||
>= req_status.req.num_tokens // offloaded_block_size
|
||||
)
|
||||
|
||||
is_eagle_unverified = (
|
||||
group_config.is_eagle_group and group_idx not in eagle_verified
|
||||
)
|
||||
|
||||
# Constrain to block-aligned boundary for this group
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
|
||||
@@ -461,15 +496,25 @@ class OffloadingConnectorScheduler:
|
||||
# we can only load less than a block, better skip
|
||||
return 0
|
||||
|
||||
num_blocks = min(
|
||||
cdiv(max_hit_size_tokens, offloaded_block_size), len(offload_keys)
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
sliding_window_size_in_blocks = (
|
||||
group_config.sliding_window_size_in_blocks
|
||||
)
|
||||
|
||||
# For eagle groups, query one extra block that will be popped.
|
||||
# We only need to increase the query size for sliding window groups.
|
||||
query_max = max_hit_size_tokens
|
||||
if is_eagle_unverified and sliding_window_size_in_blocks is not None:
|
||||
query_max = min(
|
||||
max_hit_size_tokens + offloaded_block_size,
|
||||
len(offload_keys) * offloaded_block_size,
|
||||
)
|
||||
|
||||
num_blocks = min(
|
||||
cdiv(query_max, offloaded_block_size), len(offload_keys)
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
|
||||
# end index (in the sliced offload_keys) up to which we
|
||||
# have backend-confirmed hits
|
||||
num_hit_blocks: int | None
|
||||
@@ -478,9 +523,12 @@ class OffloadingConnectorScheduler:
|
||||
offload_keys, req_status.req_context
|
||||
)
|
||||
else:
|
||||
required_window = sliding_window_size_in_blocks
|
||||
if is_eagle_unverified:
|
||||
required_window += 1
|
||||
num_hit_blocks = self._sliding_window_lookup(
|
||||
offload_keys,
|
||||
sliding_window_size_in_blocks,
|
||||
required_window,
|
||||
req_status.req_context,
|
||||
)
|
||||
if num_hit_blocks == 0:
|
||||
@@ -489,6 +537,10 @@ class OffloadingConnectorScheduler:
|
||||
if num_hit_blocks is None:
|
||||
defer_lookup = True
|
||||
else:
|
||||
if is_eagle_unverified:
|
||||
num_hit_blocks -= 1
|
||||
eagle_verified.add(group_idx)
|
||||
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens,
|
||||
offloaded_block_size * (start_block_idx + num_hit_blocks),
|
||||
@@ -500,6 +552,8 @@ class OffloadingConnectorScheduler:
|
||||
return 0
|
||||
|
||||
if new_num_hit_tokens < num_hit_tokens:
|
||||
if not group_config.is_eagle_group:
|
||||
eagle_verified.clear()
|
||||
if defer_lookup:
|
||||
# make another iteration on all groups to check
|
||||
# if we still need to defer lookup
|
||||
@@ -791,6 +845,9 @@ class OffloadingConnectorScheduler:
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
if group_config.is_eagle_group:
|
||||
num_blocks = max(0, num_blocks - 1)
|
||||
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
if num_blocks <= start_block_idx:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user