[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Claude Opus 4.8
Roger Wang
parent
ac614587f5
commit
3e6529cc0e
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (self.use_eagle).
|
||||
scheduler.use_eagle = True
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=250,
|
||||
mm_positions=mm_positions,
|
||||
)[0]
|
||||
manager = scheduler.encoder_cache_manager
|
||||
manager.allocate(request, 0)
|
||||
mm_end = 150 # offset + length
|
||||
|
||||
# Confirmed progress reaches the range end: without spec decode this frees
|
||||
# (see test below), but the drafter's +1 look-ahead still needs it.
|
||||
request.num_computed_tokens = mm_end
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == {0}
|
||||
|
||||
# One position past the range end: the +1 look-ahead has now passed it.
|
||||
request.num_computed_tokens = mm_end + 1
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for EncoderRunner.gather_mm_embeddings (model runner V2).
|
||||
|
||||
Covers the speculative-drafter encoder-cache handling: the drafter reads one
|
||||
position ahead of the target model (``draft_lookahead``). The +1 look-ahead
|
||||
feature past the processed boundary is used when its encoder output is present
|
||||
and tolerated (token-embedding fallback) when it is not, while a miss within
|
||||
the processed range still fails loudly.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(
|
||||
features: list[MultiModalFeatureSpec],
|
||||
cached: list[MultiModalFeatureSpec],
|
||||
) -> EncoderRunner:
|
||||
cache = EncoderCache()
|
||||
cache.mm_features["req0"] = features
|
||||
for f in cached:
|
||||
length = f.mm_position.length
|
||||
cache.encoder_outputs[f.identifier] = torch.arange(
|
||||
length * HIDDEN, dtype=torch.float32
|
||||
).reshape(length, HIDDEN)
|
||||
return EncoderRunner(
|
||||
model=None, # unused by gather_mm_embeddings
|
||||
max_num_tokens=64,
|
||||
hidden_size=HIDDEN,
|
||||
encoder_cache=cache,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
|
||||
# Single prefilling request, computed_prefill=0, prefill_len large.
|
||||
return runner.gather_mm_embeddings(
|
||||
req_ids=["req0"],
|
||||
total_num_scheduled_tokens=num_scheduled,
|
||||
num_scheduled_tokens=np.array([num_scheduled]),
|
||||
query_start_loc=np.array([0]),
|
||||
prefill_lens=np.array([1000]),
|
||||
computed_prefill_lens=np.array([0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
|
||||
def test_draft_lookahead_uses_boundary_feature_when_cached():
|
||||
"""The drafter's +1 look-ahead can reach the feature at offset ==
|
||||
processed_end (the next chunk). When its encoder output is already cached
|
||||
(the scheduler encoded it ahead), it is used for the look-ahead position
|
||||
rather than ignored."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
|
||||
runner = _make_runner([f0, f1], cached=[f0, f1])
|
||||
|
||||
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
|
||||
assert len(mm_embeds) == 2
|
||||
assert bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
|
||||
def test_draft_lookahead_tolerates_missing_boundary_feature():
|
||||
"""When the +1 look-ahead feature past the processed boundary is not yet
|
||||
encoded, fall back to the token embedding (the draft token is verified by
|
||||
the target) instead of raising."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
|
||||
runner = _make_runner([f0, f1], cached=[f0])
|
||||
|
||||
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
# Only f0 is gathered; f1's boundary position falls back silently.
|
||||
assert len(mm_embeds) == 1
|
||||
assert not bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 7
|
||||
|
||||
|
||||
def test_draft_lookahead_raises_on_interior_miss():
|
||||
"""A miss for a feature within the processed range (not the look-ahead
|
||||
boundary) is a real invariant violation and must fail loudly, even on the
|
||||
drafter path."""
|
||||
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
|
||||
runner = _make_runner([f0], cached=[])
|
||||
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
|
||||
def test_target_path_raises_on_encoder_cache_miss():
|
||||
"""On the target path (no look-ahead) a miss is a real invariant
|
||||
violation and must fail loudly."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
runner = _make_runner([f0], cached=[])
|
||||
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather(runner, num_scheduled=8, draft_lookahead=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_lookahead", [0, 1])
|
||||
def test_multi_request_batch_gathers_per_request(draft_lookahead):
|
||||
"""Two prefilling requests in one batch: per-request query bounds must be
|
||||
indexed by request, not applied as whole arrays."""
|
||||
a0 = _feature("a0", offset=0, length=8)
|
||||
b0 = _feature("b0", offset=0, length=8)
|
||||
cache = EncoderCache()
|
||||
cache.mm_features["req0"] = [a0]
|
||||
cache.mm_features["req1"] = [b0]
|
||||
for f in (a0, b0):
|
||||
cache.encoder_outputs[f.identifier] = torch.arange(
|
||||
f.mm_position.length * HIDDEN, dtype=torch.float32
|
||||
).reshape(f.mm_position.length, HIDDEN)
|
||||
runner = EncoderRunner(
|
||||
model=None,
|
||||
max_num_tokens=64,
|
||||
hidden_size=HIDDEN,
|
||||
encoder_cache=cache,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
mm_embeds, is_mm_embed = runner.gather_mm_embeddings(
|
||||
req_ids=["req0", "req1"],
|
||||
total_num_scheduled_tokens=16,
|
||||
num_scheduled_tokens=np.array([8, 8]),
|
||||
query_start_loc=np.array([0, 8]),
|
||||
prefill_lens=np.array([1000, 1000]),
|
||||
computed_prefill_lens=np.array([0, 0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
# Both requests contribute a feature; with the +1 skew each marks 7 of its
|
||||
# 8 positions (the skew drops one), otherwise all 8.
|
||||
assert len(mm_embeds) == 2
|
||||
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for GPUModelRunner._gather_mm_embeddings (model runner V1).
|
||||
|
||||
Mirrors tests/v1/worker/test_encoder_runner.py (the V2 runner): the EAGLE/MTP
|
||||
drafter reads one position ahead of the target (shift_computed_tokens=1). The
|
||||
+1 look-ahead feature past the processed boundary is used when its encoder
|
||||
output is present and tolerated (token-embedding fallback) when it is not,
|
||||
while a miss within the processed range still fails loudly.
|
||||
|
||||
`_gather_mm_embeddings` only uses CPU-side state, so it is exercised against a
|
||||
lightweight stub for `self` instead of a full (CUDA-only) runner.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
|
||||
encoder_cache = {
|
||||
f.identifier: torch.arange(
|
||||
f.mm_position.length * HIDDEN, dtype=torch.float32
|
||||
).reshape(f.mm_position.length, HIDDEN)
|
||||
for f in cached
|
||||
}
|
||||
req_state = SimpleNamespace(num_computed_tokens=num_computed, mm_features=features)
|
||||
runner = SimpleNamespace(
|
||||
input_batch=SimpleNamespace(req_ids=["req0"]),
|
||||
requests={"req0": req_state},
|
||||
encoder_cache=encoder_cache,
|
||||
is_multimodal_pruning_enabled=False,
|
||||
uses_mrope=False,
|
||||
)
|
||||
scheduler_output = SimpleNamespace(
|
||||
total_num_scheduled_tokens=num_scheduled,
|
||||
num_scheduled_tokens={"req0": num_scheduled},
|
||||
)
|
||||
return GPUModelRunner._gather_mm_embeddings(
|
||||
runner, scheduler_output, shift_computed_tokens=shift
|
||||
)
|
||||
|
||||
|
||||
def test_draft_shift_uses_boundary_feature_when_cached():
|
||||
"""The drafter's +1 look-ahead reaches the feature at offset ==
|
||||
processed_end; when it is already cached it is used for the look-ahead
|
||||
position rather than ignored."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
|
||||
mm_embeds, is_mm_embed = _gather([f0, f1], [f0, f1], num_scheduled=8, shift=1)
|
||||
|
||||
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
|
||||
assert len(mm_embeds) == 2
|
||||
assert bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
|
||||
def test_draft_shift_tolerates_missing_boundary_feature():
|
||||
"""When the +1 look-ahead feature past the processed boundary is not yet
|
||||
encoded, fall back to the token embedding instead of raising."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
|
||||
mm_embeds, is_mm_embed = _gather([f0, f1], [f0], num_scheduled=8, shift=1)
|
||||
|
||||
assert len(mm_embeds) == 1 # only f0; f1's boundary position falls back
|
||||
assert not bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 7
|
||||
|
||||
|
||||
def test_draft_shift_raises_on_interior_miss():
|
||||
"""A miss for a feature within the processed range (not the look-ahead
|
||||
boundary) is a real invariant violation, even on the drafter path."""
|
||||
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather([f0], [], num_scheduled=8, shift=1)
|
||||
|
||||
|
||||
def test_target_path_raises_on_encoder_cache_miss():
|
||||
"""On the target path (no shift) a miss is a real invariant violation."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather([f0], [], num_scheduled=8, shift=0)
|
||||
Reference in New Issue
Block a user