[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Nick Hill
2026-06-22 18:14:02 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Roger Wang
parent ac614587f5
commit 3e6529cc0e
7 changed files with 331 additions and 18 deletions
+32
View File
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_defers_for_eagle_lookahead():
"""With EAGLE speculative decoding, the encoder input is retained one extra
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (self.use_eagle).
scheduler.use_eagle = True
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
num_requests=1,
num_tokens=250,
mm_positions=mm_positions,
)[0]
manager = scheduler.encoder_cache_manager
manager.allocate(request, 0)
mm_end = 150 # offset + length
# Confirmed progress reaches the range end: without spec decode this frees
# (see test below), but the drafter's +1 look-ahead still needs it.
request.num_computed_tokens = mm_end
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == {0}
# One position past the range end: the +1 look-ahead has now passed it.
request.num_computed_tokens = mm_end + 1
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before."""
+157
View File
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for EncoderRunner.gather_mm_embeddings (model runner V2).
Covers the speculative-drafter encoder-cache handling: the drafter reads one
position ahead of the target model (``draft_lookahead``). The +1 look-ahead
feature past the processed boundary is used when its encoder output is present
and tolerated (token-embedding fallback) when it is not, while a miss within
the processed range still fails loudly.
"""
import numpy as np
import pytest
import torch
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
pytestmark = pytest.mark.cpu_test
HIDDEN = 4
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=None,
modality="image",
identifier=identifier,
mm_position=PlaceholderRange(offset=offset, length=length),
)
def _make_runner(
features: list[MultiModalFeatureSpec],
cached: list[MultiModalFeatureSpec],
) -> EncoderRunner:
cache = EncoderCache()
cache.mm_features["req0"] = features
for f in cached:
length = f.mm_position.length
cache.encoder_outputs[f.identifier] = torch.arange(
length * HIDDEN, dtype=torch.float32
).reshape(length, HIDDEN)
return EncoderRunner(
model=None, # unused by gather_mm_embeddings
max_num_tokens=64,
hidden_size=HIDDEN,
encoder_cache=cache,
dtype=torch.float32,
device=torch.device("cpu"),
)
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
# Single prefilling request, computed_prefill=0, prefill_len large.
return runner.gather_mm_embeddings(
req_ids=["req0"],
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens=np.array([num_scheduled]),
query_start_loc=np.array([0]),
prefill_lens=np.array([1000]),
computed_prefill_lens=np.array([0]),
draft_lookahead=draft_lookahead,
)
def test_draft_lookahead_uses_boundary_feature_when_cached():
"""The drafter's +1 look-ahead can reach the feature at offset ==
processed_end (the next chunk). When its encoder output is already cached
(the scheduler encoded it ahead), it is used for the look-ahead position
rather than ignored."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
runner = _make_runner([f0, f1], cached=[f0, f1])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
def test_draft_lookahead_tolerates_missing_boundary_feature():
"""When the +1 look-ahead feature past the processed boundary is not yet
encoded, fall back to the token embedding (the draft token is verified by
the target) instead of raising."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
runner = _make_runner([f0, f1], cached=[f0])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
# Only f0 is gathered; f1's boundary position falls back silently.
assert len(mm_embeds) == 1
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
def test_draft_lookahead_raises_on_interior_miss():
"""A miss for a feature within the processed range (not the look-ahead
boundary) is a real invariant violation and must fail loudly, even on the
drafter path."""
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
runner = _make_runner([f0], cached=[])
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather(runner, num_scheduled=8, draft_lookahead=1)
def test_target_path_raises_on_encoder_cache_miss():
"""On the target path (no look-ahead) a miss is a real invariant
violation and must fail loudly."""
f0 = _feature("h0", offset=0, length=8)
runner = _make_runner([f0], cached=[])
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather(runner, num_scheduled=8, draft_lookahead=0)
@pytest.mark.parametrize("draft_lookahead", [0, 1])
def test_multi_request_batch_gathers_per_request(draft_lookahead):
"""Two prefilling requests in one batch: per-request query bounds must be
indexed by request, not applied as whole arrays."""
a0 = _feature("a0", offset=0, length=8)
b0 = _feature("b0", offset=0, length=8)
cache = EncoderCache()
cache.mm_features["req0"] = [a0]
cache.mm_features["req1"] = [b0]
for f in (a0, b0):
cache.encoder_outputs[f.identifier] = torch.arange(
f.mm_position.length * HIDDEN, dtype=torch.float32
).reshape(f.mm_position.length, HIDDEN)
runner = EncoderRunner(
model=None,
max_num_tokens=64,
hidden_size=HIDDEN,
encoder_cache=cache,
dtype=torch.float32,
device=torch.device("cpu"),
)
mm_embeds, is_mm_embed = runner.gather_mm_embeddings(
req_ids=["req0", "req1"],
total_num_scheduled_tokens=16,
num_scheduled_tokens=np.array([8, 8]),
query_start_loc=np.array([0, 8]),
prefill_lens=np.array([1000, 1000]),
computed_prefill_lens=np.array([0, 0]),
draft_lookahead=draft_lookahead,
)
# Both requests contribute a feature; with the +1 skew each marks 7 of its
# 8 positions (the skew drops one), otherwise all 8.
assert len(mm_embeds) == 2
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GPUModelRunner._gather_mm_embeddings (model runner V1).
Mirrors tests/v1/worker/test_encoder_runner.py (the V2 runner): the EAGLE/MTP
drafter reads one position ahead of the target (shift_computed_tokens=1). The
+1 look-ahead feature past the processed boundary is used when its encoder
output is present and tolerated (token-embedding fallback) when it is not,
while a miss within the processed range still fails loudly.
`_gather_mm_embeddings` only uses CPU-side state, so it is exercised against a
lightweight stub for `self` instead of a full (CUDA-only) runner.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
pytestmark = pytest.mark.cpu_test
HIDDEN = 4
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
return MultiModalFeatureSpec(
data=None,
modality="image",
identifier=identifier,
mm_position=PlaceholderRange(offset=offset, length=length),
)
def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
encoder_cache = {
f.identifier: torch.arange(
f.mm_position.length * HIDDEN, dtype=torch.float32
).reshape(f.mm_position.length, HIDDEN)
for f in cached
}
req_state = SimpleNamespace(num_computed_tokens=num_computed, mm_features=features)
runner = SimpleNamespace(
input_batch=SimpleNamespace(req_ids=["req0"]),
requests={"req0": req_state},
encoder_cache=encoder_cache,
is_multimodal_pruning_enabled=False,
uses_mrope=False,
)
scheduler_output = SimpleNamespace(
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens={"req0": num_scheduled},
)
return GPUModelRunner._gather_mm_embeddings(
runner, scheduler_output, shift_computed_tokens=shift
)
def test_draft_shift_uses_boundary_feature_when_cached():
"""The drafter's +1 look-ahead reaches the feature at offset ==
processed_end; when it is already cached it is used for the look-ahead
position rather than ignored."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
mm_embeds, is_mm_embed = _gather([f0, f1], [f0, f1], num_scheduled=8, shift=1)
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
def test_draft_shift_tolerates_missing_boundary_feature():
"""When the +1 look-ahead feature past the processed boundary is not yet
encoded, fall back to the token embedding instead of raising."""
f0 = _feature("h0", offset=0, length=8)
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
mm_embeds, is_mm_embed = _gather([f0, f1], [f0], num_scheduled=8, shift=1)
assert len(mm_embeds) == 1 # only f0; f1's boundary position falls back
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
def test_draft_shift_raises_on_interior_miss():
"""A miss for a feature within the processed range (not the look-ahead
boundary) is a real invariant violation, even on the drafter path."""
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather([f0], [], num_scheduled=8, shift=1)
def test_target_path_raises_on_encoder_cache_miss():
"""On the target path (no shift) a miss is a real invariant violation."""
f0 = _feature("h0", offset=0, length=8)
with pytest.raises(RuntimeError, match="Encoder cache miss"):
_gather([f0], [], num_scheduled=8, shift=0)