[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Claude Opus 4.8
Roger Wang
parent
ac614587f5
commit
3e6529cc0e
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (self.use_eagle).
|
||||
scheduler.use_eagle = True
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=250,
|
||||
mm_positions=mm_positions,
|
||||
)[0]
|
||||
manager = scheduler.encoder_cache_manager
|
||||
manager.allocate(request, 0)
|
||||
mm_end = 150 # offset + length
|
||||
|
||||
# Confirmed progress reaches the range end: without spec decode this frees
|
||||
# (see test below), but the drafter's +1 look-ahead still needs it.
|
||||
request.num_computed_tokens = mm_end
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == {0}
|
||||
|
||||
# One position past the range end: the +1 look-ahead has now passed it.
|
||||
request.num_computed_tokens = mm_end + 1
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for EncoderRunner.gather_mm_embeddings (model runner V2).
|
||||
|
||||
Covers the speculative-drafter encoder-cache handling: the drafter reads one
|
||||
position ahead of the target model (``draft_lookahead``). The +1 look-ahead
|
||||
feature past the processed boundary is used when its encoder output is present
|
||||
and tolerated (token-embedding fallback) when it is not, while a miss within
|
||||
the processed range still fails loudly.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(
|
||||
features: list[MultiModalFeatureSpec],
|
||||
cached: list[MultiModalFeatureSpec],
|
||||
) -> EncoderRunner:
|
||||
cache = EncoderCache()
|
||||
cache.mm_features["req0"] = features
|
||||
for f in cached:
|
||||
length = f.mm_position.length
|
||||
cache.encoder_outputs[f.identifier] = torch.arange(
|
||||
length * HIDDEN, dtype=torch.float32
|
||||
).reshape(length, HIDDEN)
|
||||
return EncoderRunner(
|
||||
model=None, # unused by gather_mm_embeddings
|
||||
max_num_tokens=64,
|
||||
hidden_size=HIDDEN,
|
||||
encoder_cache=cache,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
|
||||
# Single prefilling request, computed_prefill=0, prefill_len large.
|
||||
return runner.gather_mm_embeddings(
|
||||
req_ids=["req0"],
|
||||
total_num_scheduled_tokens=num_scheduled,
|
||||
num_scheduled_tokens=np.array([num_scheduled]),
|
||||
query_start_loc=np.array([0]),
|
||||
prefill_lens=np.array([1000]),
|
||||
computed_prefill_lens=np.array([0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
|
||||
def test_draft_lookahead_uses_boundary_feature_when_cached():
|
||||
"""The drafter's +1 look-ahead can reach the feature at offset ==
|
||||
processed_end (the next chunk). When its encoder output is already cached
|
||||
(the scheduler encoded it ahead), it is used for the look-ahead position
|
||||
rather than ignored."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
|
||||
runner = _make_runner([f0, f1], cached=[f0, f1])
|
||||
|
||||
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
|
||||
assert len(mm_embeds) == 2
|
||||
assert bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
|
||||
def test_draft_lookahead_tolerates_missing_boundary_feature():
|
||||
"""When the +1 look-ahead feature past the processed boundary is not yet
|
||||
encoded, fall back to the token embedding (the draft token is verified by
|
||||
the target) instead of raising."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
|
||||
runner = _make_runner([f0, f1], cached=[f0])
|
||||
|
||||
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
# Only f0 is gathered; f1's boundary position falls back silently.
|
||||
assert len(mm_embeds) == 1
|
||||
assert not bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 7
|
||||
|
||||
|
||||
def test_draft_lookahead_raises_on_interior_miss():
|
||||
"""A miss for a feature within the processed range (not the look-ahead
|
||||
boundary) is a real invariant violation and must fail loudly, even on the
|
||||
drafter path."""
|
||||
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
|
||||
runner = _make_runner([f0], cached=[])
|
||||
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather(runner, num_scheduled=8, draft_lookahead=1)
|
||||
|
||||
|
||||
def test_target_path_raises_on_encoder_cache_miss():
|
||||
"""On the target path (no look-ahead) a miss is a real invariant
|
||||
violation and must fail loudly."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
runner = _make_runner([f0], cached=[])
|
||||
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather(runner, num_scheduled=8, draft_lookahead=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_lookahead", [0, 1])
|
||||
def test_multi_request_batch_gathers_per_request(draft_lookahead):
|
||||
"""Two prefilling requests in one batch: per-request query bounds must be
|
||||
indexed by request, not applied as whole arrays."""
|
||||
a0 = _feature("a0", offset=0, length=8)
|
||||
b0 = _feature("b0", offset=0, length=8)
|
||||
cache = EncoderCache()
|
||||
cache.mm_features["req0"] = [a0]
|
||||
cache.mm_features["req1"] = [b0]
|
||||
for f in (a0, b0):
|
||||
cache.encoder_outputs[f.identifier] = torch.arange(
|
||||
f.mm_position.length * HIDDEN, dtype=torch.float32
|
||||
).reshape(f.mm_position.length, HIDDEN)
|
||||
runner = EncoderRunner(
|
||||
model=None,
|
||||
max_num_tokens=64,
|
||||
hidden_size=HIDDEN,
|
||||
encoder_cache=cache,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
mm_embeds, is_mm_embed = runner.gather_mm_embeddings(
|
||||
req_ids=["req0", "req1"],
|
||||
total_num_scheduled_tokens=16,
|
||||
num_scheduled_tokens=np.array([8, 8]),
|
||||
query_start_loc=np.array([0, 8]),
|
||||
prefill_lens=np.array([1000, 1000]),
|
||||
computed_prefill_lens=np.array([0, 0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
# Both requests contribute a feature; with the +1 skew each marks 7 of its
|
||||
# 8 positions (the skew drops one), otherwise all 8.
|
||||
assert len(mm_embeds) == 2
|
||||
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for GPUModelRunner._gather_mm_embeddings (model runner V1).
|
||||
|
||||
Mirrors tests/v1/worker/test_encoder_runner.py (the V2 runner): the EAGLE/MTP
|
||||
drafter reads one position ahead of the target (shift_computed_tokens=1). The
|
||||
+1 look-ahead feature past the processed boundary is used when its encoder
|
||||
output is present and tolerated (token-embedding fallback) when it is not,
|
||||
while a miss within the processed range still fails loudly.
|
||||
|
||||
`_gather_mm_embeddings` only uses CPU-side state, so it is exercised against a
|
||||
lightweight stub for `self` instead of a full (CUDA-only) runner.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
def _feature(identifier: str, offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier=identifier,
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
|
||||
encoder_cache = {
|
||||
f.identifier: torch.arange(
|
||||
f.mm_position.length * HIDDEN, dtype=torch.float32
|
||||
).reshape(f.mm_position.length, HIDDEN)
|
||||
for f in cached
|
||||
}
|
||||
req_state = SimpleNamespace(num_computed_tokens=num_computed, mm_features=features)
|
||||
runner = SimpleNamespace(
|
||||
input_batch=SimpleNamespace(req_ids=["req0"]),
|
||||
requests={"req0": req_state},
|
||||
encoder_cache=encoder_cache,
|
||||
is_multimodal_pruning_enabled=False,
|
||||
uses_mrope=False,
|
||||
)
|
||||
scheduler_output = SimpleNamespace(
|
||||
total_num_scheduled_tokens=num_scheduled,
|
||||
num_scheduled_tokens={"req0": num_scheduled},
|
||||
)
|
||||
return GPUModelRunner._gather_mm_embeddings(
|
||||
runner, scheduler_output, shift_computed_tokens=shift
|
||||
)
|
||||
|
||||
|
||||
def test_draft_shift_uses_boundary_feature_when_cached():
|
||||
"""The drafter's +1 look-ahead reaches the feature at offset ==
|
||||
processed_end; when it is already cached it is used for the look-ahead
|
||||
position rather than ignored."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # starts exactly at processed_end
|
||||
mm_embeds, is_mm_embed = _gather([f0, f1], [f0, f1], num_scheduled=8, shift=1)
|
||||
|
||||
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
|
||||
assert len(mm_embeds) == 2
|
||||
assert bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
|
||||
def test_draft_shift_tolerates_missing_boundary_feature():
|
||||
"""When the +1 look-ahead feature past the processed boundary is not yet
|
||||
encoded, fall back to the token embedding instead of raising."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
f1 = _feature("h1", offset=8, length=8) # boundary feature, not cached
|
||||
mm_embeds, is_mm_embed = _gather([f0, f1], [f0], num_scheduled=8, shift=1)
|
||||
|
||||
assert len(mm_embeds) == 1 # only f0; f1's boundary position falls back
|
||||
assert not bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 7
|
||||
|
||||
|
||||
def test_draft_shift_raises_on_interior_miss():
|
||||
"""A miss for a feature within the processed range (not the look-ahead
|
||||
boundary) is a real invariant violation, even on the drafter path."""
|
||||
f0 = _feature("h0", offset=0, length=8) # interior, within processed range
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather([f0], [], num_scheduled=8, shift=1)
|
||||
|
||||
|
||||
def test_target_path_raises_on_encoder_cache_miss():
|
||||
"""On the target path (no shift) a miss is a real invariant violation."""
|
||||
f0 = _feature("h0", offset=0, length=8)
|
||||
with pytest.raises(RuntimeError, match="Encoder cache miss"):
|
||||
_gather([f0], [], num_scheduled=8, shift=0)
|
||||
@@ -1872,6 +1872,11 @@ class Scheduler(SchedulerInterface):
|
||||
if not cached_encoder_input_ids:
|
||||
return
|
||||
|
||||
# Defer the free by the drafter's look-ahead so an entry stays
|
||||
# referenced until the drafter's +1 read has also passed it, mirroring
|
||||
# the shift the encoder scheduling path applies.
|
||||
spec_lookahead = 1 if self.use_eagle else 0
|
||||
|
||||
# Here, we use list(set) to avoid modifying the set while iterating
|
||||
# over it.
|
||||
for input_id in list(cached_encoder_input_ids):
|
||||
@@ -1884,13 +1889,12 @@ class Scheduler(SchedulerInterface):
|
||||
# KVs have been calculated and cached already.
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
elif (
|
||||
start_pos + num_tokens
|
||||
start_pos + num_tokens + spec_lookahead
|
||||
<= request.num_computed_tokens - request.num_output_placeholders
|
||||
):
|
||||
# The encoder output is already processed and stored in the
|
||||
# decoder's KV cache, and progress is far enough past the
|
||||
# placeholder range that no pending draft-token rejection can
|
||||
# roll num_computed_tokens back into it.
|
||||
# Processed, stored in the decoder KV cache, and far enough past
|
||||
# the placeholder range (plus the drafter's look-ahead) that no
|
||||
# rejection or drafter gather can reference it.
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
|
||||
def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None:
|
||||
|
||||
@@ -68,15 +68,19 @@ class EncoderRunner:
|
||||
query_start_loc: np.ndarray,
|
||||
prefill_lens: np.ndarray,
|
||||
computed_prefill_lens: np.ndarray,
|
||||
draft_lookahead: int = 0,
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
is_prefilling = (computed_prefill_lens < prefill_lens).tolist()
|
||||
all_decode = not any(is_prefilling)
|
||||
if all_decode:
|
||||
if draft_lookahead:
|
||||
computed_prefill_lens = computed_prefill_lens + draft_lookahead
|
||||
|
||||
is_prefilling_np = computed_prefill_lens < prefill_lens
|
||||
if not is_prefilling_np.any():
|
||||
# All decode requests, so no need to gather any embeddings.
|
||||
return [], torch.zeros(
|
||||
total_num_scheduled_tokens, dtype=torch.bool, device=self.device
|
||||
)
|
||||
|
||||
is_prefilling = is_prefilling_np.tolist()
|
||||
query_start = computed_prefill_lens.tolist()
|
||||
query_end = (computed_prefill_lens + num_scheduled_tokens).tolist()
|
||||
|
||||
@@ -89,11 +93,12 @@ class EncoderRunner:
|
||||
# OPTIMIZATION: Skip decode requests.
|
||||
continue
|
||||
|
||||
cur_query_start = query_start[i]
|
||||
cur_query_end = query_end[i]
|
||||
|
||||
mm_features = self.encoder_cache.mm_features[req_id]
|
||||
lo, hi = get_mm_features_in_window(
|
||||
mm_features,
|
||||
start=query_start[i],
|
||||
end=query_end[i],
|
||||
mm_features, start=cur_query_start, end=cur_query_end
|
||||
)
|
||||
for idx in range(lo, hi):
|
||||
mm_feature = mm_features[idx]
|
||||
@@ -101,8 +106,8 @@ class EncoderRunner:
|
||||
start_pos = pos_info.offset
|
||||
num_encoder_tokens = pos_info.length
|
||||
|
||||
start_idx = max(query_start[i] - start_pos, 0)
|
||||
end_idx = min(query_end[i] - start_pos, num_encoder_tokens)
|
||||
start_idx = max(cur_query_start - start_pos, 0)
|
||||
end_idx = min(cur_query_end - start_pos, num_encoder_tokens)
|
||||
assert start_idx < end_idx
|
||||
curr_embeds_start, curr_embeds_end = (
|
||||
pos_info.get_embeds_indices_in_range(start_idx, end_idx)
|
||||
@@ -114,7 +119,13 @@ class EncoderRunner:
|
||||
|
||||
mm_hash = mm_feature.identifier
|
||||
encoder_output = self.encoder_cache.encoder_outputs.get(mm_hash, None)
|
||||
assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
|
||||
if encoder_output is None:
|
||||
# A feature starting at/after the processed boundary is only
|
||||
# reached via the drafter's +1 look-ahead and might not be
|
||||
# encoded yet; fall back to the token embedding for drafting.
|
||||
if start_pos + draft_lookahead >= cur_query_end:
|
||||
continue
|
||||
raise RuntimeError(f"Encoder cache miss for {mm_hash}.")
|
||||
|
||||
if (is_embed := pos_info.is_embed) is not None:
|
||||
is_embed = is_embed[start_idx:end_idx]
|
||||
@@ -122,7 +133,7 @@ class EncoderRunner:
|
||||
else:
|
||||
mm_embeds_item = encoder_output[start_idx:end_idx]
|
||||
|
||||
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
|
||||
req_start_pos = query_start_loc[i] + start_pos - cur_query_start
|
||||
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
|
||||
True if is_embed is None else is_embed
|
||||
)
|
||||
|
||||
@@ -1409,8 +1409,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
# +1 to consider the skew in eagle
|
||||
input_batch.num_computed_prefill_tokens_np + 1,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
# The EAGLE/MTP drafter reads one position ahead of the target.
|
||||
draft_lookahead=1,
|
||||
)
|
||||
|
||||
# Postprocess results and update request states.
|
||||
|
||||
@@ -3152,7 +3152,16 @@ class GPUModelRunner(
|
||||
|
||||
mm_hash = mm_feature.identifier
|
||||
encoder_output = self.encoder_cache.get(mm_hash, None)
|
||||
assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
|
||||
if encoder_output is None:
|
||||
# A feature starting at/after the processed boundary is only
|
||||
# reached via the drafter's +1 look-ahead and might not be
|
||||
# encoded yet; fall back to the token embedding for drafting.
|
||||
if (
|
||||
start_pos
|
||||
>= req_state.num_computed_tokens + num_scheduled_tokens
|
||||
):
|
||||
continue
|
||||
raise RuntimeError(f"Encoder cache miss for {mm_hash}.")
|
||||
|
||||
if (is_embed := pos_info.is_embed) is not None:
|
||||
is_embed = is_embed[start_idx:end_idx]
|
||||
|
||||
Reference in New Issue
Block a user