[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Nick Hill
2026-06-22 18:14:02 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Roger Wang
parent ac614587f5
commit 3e6529cc0e
7 changed files with 331 additions and 18 deletions
+32
View File
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_defers_for_eagle_lookahead():
"""With EAGLE speculative decoding, the encoder input is retained one extra
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (self.use_eagle).
scheduler.use_eagle = True
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
num_requests=1,
num_tokens=250,
mm_positions=mm_positions,
)[0]
manager = scheduler.encoder_cache_manager
manager.allocate(request, 0)
mm_end = 150 # offset + length
# Confirmed progress reaches the range end: without spec decode this frees
# (see test below), but the drafter's +1 look-ahead still needs it.
request.num_computed_tokens = mm_end
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == {0}
# One position past the range end: the +1 look-ahead has now passed it.
request.num_computed_tokens = mm_end + 1
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before."""
+157
View File
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for EncoderRunner.gather_mm_embeddings (model runner V2).
Covers the speculative-drafter encoder-cache handling: the drafter reads one
position ahead of the target model (``draft_lookahead``). The +1 look-ahead
feature past the processed boundary is used when its encoder output is present
and tolerated (token-embedding fallback) when it is not, while a miss within
the processed range still fails loudly.
"""
import numpy as np
import pytest
import torch
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
pytestmark = pytest.mark.cpu_test
HIDDEN = 4
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=None,
modality="image",
identifier=identifier,
mm_position=PlaceholderRange(offset=offset, length=length),
)
def _make_runner(
features: list[MultiModalFeatureSpec],
cached: list[MultiModalFeatureSpec],
) -> EncoderRunner:
cache = EncoderCache()
cache.mm_features["req0"] = features
for f in cached:
length = f.mm_position.length
cache.encoder_outputs[f.identifier] = torch.arange(
length * HIDDEN, dtype=torch.float32
).reshape(length, HIDDEN)
return EncoderRunner(
model=None, # unused by gather_mm_embeddings
max_num_tokens=64,
hidden_size=HIDDEN,
encoder_cache=cache,
dtype=torch.float32,
device=torch.device("cpu"),
)
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
# Single prefilling request, computed_prefill=0, prefill_len large.
return runner.gather_mm_embeddings(
req_ids=["req0"],
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens=np.array([num_scheduled]),
query_start_loc=np.array([0]),
prefill_lens=np.array([1000]),
computed_prefill_lens=np.array([0]),
draft_lookahead=draft_lookahead,
)
def test_draft_lookahead_uses_boundary_feature_when_cached():
"""The drafter's +1 look-ahead can reach the feature at offset ==
processed_end (the next chunk). When its encoder output is already cached
(the scheduler encoded it ahead), it is used for the look-ahead position
rather than ignored."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
runner = _make_runner([f0, f1], cached=[f0, f1])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
def test_draft_lookahead_tolerates_missing_boundary_feature():
"""When the +1 look-ahead feature past the processed boundary is not yet
encoded, fall back to the token embedding (the draft token is verified by
the target) instead of raising."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
runner = _make_runner([f0, f1], cached=[f0])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
# Only f0 is gathered; f1's boundary position falls back silently.
assert len(mm_embeds) == 1
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
def test_draft_lookahead_raises_on_interior_miss():
"""A miss for a feature within the processed range (not the look-ahead
boundary) is a real invariant violation and must fail loudly, even on the
drafter path."""
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
runner = _make_runner([f0], cached=[])
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather(runner, num_scheduled=8, draft_lookahead=1)
def test_target_path_raises_on_encoder_cache_miss():
"""On the target path (no look-ahead) a miss is a real invariant
violation and must fail loudly."""
f0 = _feature("h0", offset=0, length=8)
runner = _make_runner([f0], cached=[])
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather(runner, num_scheduled=8, draft_lookahead=0)
@pytest.mark.parametrize("draft_lookahead", [0, 1])
def test_multi_request_batch_gathers_per_request(draft_lookahead):
"""Two prefilling requests in one batch: per-request query bounds must be
indexed by request, not applied as whole arrays."""
a0 = _feature("a0", offset=0, length=8)
b0 = _feature("b0", offset=0, length=8)
cache = EncoderCache()
cache.mm_features["req0"] = [a0]
cache.mm_features["req1"] = [b0]
for f in (a0, b0):
cache.encoder_outputs[f.identifier] = torch.arange(
f.mm_position.length * HIDDEN, dtype=torch.float32
).reshape(f.mm_position.length, HIDDEN)
runner = EncoderRunner(
model=None,
max_num_tokens=64,
hidden_size=HIDDEN,
encoder_cache=cache,
dtype=torch.float32,
device=torch.device("cpu"),
)
mm_embeds, is_mm_embed = runner.gather_mm_embeddings(
req_ids=["req0", "req1"],
total_num_scheduled_tokens=16,
num_scheduled_tokens=np.array([8, 8]),
query_start_loc=np.array([0, 8]),
prefill_lens=np.array([1000, 1000]),
computed_prefill_lens=np.array([0, 0]),
draft_lookahead=draft_lookahead,
)
# Both requests contribute a feature; with the +1 skew each marks 7 of its
# 8 positions (the skew drops one), otherwise all 8.
assert len(mm_embeds) == 2
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GPUModelRunner._gather_mm_embeddings (model runner V1).
Mirrors tests/v1/worker/test_encoder_runner.py (the V2 runner): the EAGLE/MTP
drafter reads one position ahead of the target (shift_computed_tokens=1). The
+1 look-ahead feature past the processed boundary is used when its encoder
output is present and tolerated (token-embedding fallback) when it is not,
while a miss within the processed range still fails loudly.
`_gather_mm_embeddings` only uses CPU-side state, so it is exercised against a
lightweight stub for `self` instead of a full (CUDA-only) runner.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
pytestmark = pytest.mark.cpu_test
HIDDEN = 4
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=None,
modality="image",
identifier=identifier,
mm_position=PlaceholderRange(offset=offset, length=length),
)
def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
encoder_cache = {
f.identifier: torch.arange(
f.mm_position.length * HIDDEN, dtype=torch.float32
).reshape(f.mm_position.length, HIDDEN)
for f in cached
}
req_state = SimpleNamespace(num_computed_tokens=num_computed, mm_features=features)
runner = SimpleNamespace(
input_batch=SimpleNamespace(req_ids=["req0"]),
requests={"req0": req_state},
encoder_cache=encoder_cache,
is_multimodal_pruning_enabled=False,
uses_mrope=False,
)
scheduler_output = SimpleNamespace(
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens={"req0": num_scheduled},
)
return GPUModelRunner._gather_mm_embeddings(
runner, scheduler_output, shift_computed_tokens=shift
)
def test_draft_shift_uses_boundary_feature_when_cached():
"""The drafter's +1 look-ahead reaches the feature at offset ==
processed_end; when it is already cached it is used for the look-ahead
position rather than ignored."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
mm_embeds, is_mm_embed = _gather([f0, f1], [f0, f1], num_scheduled=8, shift=1)
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
def test_draft_shift_tolerates_missing_boundary_feature():
"""When the +1 look-ahead feature past the processed boundary is not yet
encoded, fall back to the token embedding instead of raising."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
mm_embeds, is_mm_embed = _gather([f0, f1], [f0], num_scheduled=8, shift=1)
assert len(mm_embeds) == 1 # only f0; f1's boundary position falls back
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
def test_draft_shift_raises_on_interior_miss():
"""A miss for a feature within the processed range (not the look-ahead
boundary) is a real invariant violation, even on the drafter path."""
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather([f0], [], num_scheduled=8, shift=1)
def test_target_path_raises_on_encoder_cache_miss():
"""On the target path (no shift) a miss is a real invariant violation."""
f0 = _feature("h0", offset=0, length=8)
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather([f0], [], num_scheduled=8, shift=0)
+9 -5
View File
@@ -1872,6 +1872,11 @@ class Scheduler(SchedulerInterface):
if not cached_encoder_input_ids:
return
# Defer the free by the drafter's look-ahead so an entry stays
# referenced until the drafter's +1 read has also passed it, mirroring
# the shift the encoder scheduling path applies.
spec_lookahead = 1 if self.use_eagle else 0
# Here, we use list(set) to avoid modifying the set while iterating
# over it.
for input_id in list(cached_encoder_input_ids):
@@ -1884,13 +1889,12 @@ class Scheduler(SchedulerInterface):
# KVs have been calculated and cached already.
self.encoder_cache_manager.free_encoder_input(request, input_id)
elif (
start_pos + num_tokens
start_pos + num_tokens + spec_lookahead
<= request.num_computed_tokens - request.num_output_placeholders
):
# The encoder output is already processed and stored in the
# decoder's KV cache, and progress is far enough past the
# placeholder range that no pending draft-token rejection can
# roll num_computed_tokens back into it.
# Processed, stored in the decoder KV cache, and far enough past
# the placeholder range (plus the drafter's look-ahead) that no
# rejection or drafter gather can reference it.
self.encoder_cache_manager.free_encoder_input(request, input_id)
def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None:
+21 -10
View File
@@ -68,15 +68,19 @@ class EncoderRunner:
query_start_loc: np.ndarray,
prefill_lens: np.ndarray,
computed_prefill_lens: np.ndarray,
draft_lookahead: int = 0,
) -> tuple[list[torch.Tensor], torch.Tensor]:
is_prefilling = (computed_prefill_lens < prefill_lens).tolist()
all_decode = not any(is_prefilling)
if all_decode:
if draft_lookahead:
computed_prefill_lens = computed_prefill_lens + draft_lookahead
is_prefilling_np = computed_prefill_lens < prefill_lens
if not is_prefilling_np.any():
# All decode requests, so no need to gather any embeddings.
return [], torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device=self.device
)
is_prefilling = is_prefilling_np.tolist()
query_start = computed_prefill_lens.tolist()
query_end = (computed_prefill_lens + num_scheduled_tokens).tolist()
@@ -89,11 +93,12 @@ class EncoderRunner:
# OPTIMIZATION: Skip decode requests.
continue
cur_query_start = query_start[i]
cur_query_end = query_end[i]
mm_features = self.encoder_cache.mm_features[req_id]
lo, hi = get_mm_features_in_window(
mm_features,
start=query_start[i],
end=query_end[i],
mm_features, start=cur_query_start, end=cur_query_end
)
for idx in range(lo, hi):
mm_feature = mm_features[idx]
@@ -101,8 +106,8 @@ class EncoderRunner:
start_pos = pos_info.offset
num_encoder_tokens = pos_info.length
start_idx = max(query_start[i] - start_pos, 0)
end_idx = min(query_end[i] - start_pos, num_encoder_tokens)
start_idx = max(cur_query_start - start_pos, 0)
end_idx = min(cur_query_end - start_pos, num_encoder_tokens)
assert start_idx < end_idx
curr_embeds_start, curr_embeds_end = (
pos_info.get_embeds_indices_in_range(start_idx, end_idx)
@@ -114,7 +119,13 @@ class EncoderRunner:
mm_hash = mm_feature.identifier
encoder_output = self.encoder_cache.encoder_outputs.get(mm_hash, None)
assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
if encoder_output is None:
# A feature starting at/after the processed boundary is only
# reached via the drafter's +1 look-ahead and might not be
# encoded yet; fall back to the token embedding for drafting.
if start_pos + draft_lookahead >= cur_query_end:
continue
raise RuntimeError(f"Encoder cache miss for {mm_hash}.")
if (is_embed := pos_info.is_embed) is not None:
is_embed = is_embed[start_idx:end_idx]
@@ -122,7 +133,7 @@ class EncoderRunner:
else:
mm_embeds_item = encoder_output[start_idx:end_idx]
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
req_start_pos = query_start_loc[i] + start_pos - cur_query_start
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
True if is_embed is None else is_embed
)
+3 -2
View File
@@ -1409,8 +1409,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
# +1 to consider the skew in eagle
input_batch.num_computed_prefill_tokens_np + 1,
input_batch.num_computed_prefill_tokens_np,
# The EAGLE/MTP drafter reads one position ahead of the target.
draft_lookahead=1,
)
# Postprocess results and update request states.
+10 -1
View File
@@ -3152,7 +3152,16 @@ class GPUModelRunner(
mm_hash = mm_feature.identifier
encoder_output = self.encoder_cache.get(mm_hash, None)
assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
if encoder_output is None:
# A feature starting at/after the processed boundary is only
# reached via the drafter's +1 look-ahead and might not be
# encoded yet; fall back to the token embedding for drafting.
if (
start_pos
>= req_state.num_computed_tokens + num_scheduled_tokens
):
continue
raise RuntimeError(f"Encoder cache miss for {mm_hash}.")
if (is_embed := pos_info.is_embed) is not None:
is_embed = is_embed[start_idx:end_idx]