[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Nick Hill
2026-06-22 18:14:02 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Roger Wang
parent ac614587f5
commit 3e6529cc0e
7 changed files with 331 additions and 18 deletions
+32
View File
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_defers_for_eagle_lookahead():
"""With EAGLE speculative decoding, the encoder input is retained one extra
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (self.use_eagle).
scheduler.use_eagle = True
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
num_requests=1,
num_tokens=250,
mm_positions=mm_positions,
)[0]
manager = scheduler.encoder_cache_manager
manager.allocate(request, 0)
mm_end = 150 # offset + length
# Confirmed progress reaches the range end: without spec decode this frees
# (see test below), but the drafter's +1 look-ahead still needs it.
request.num_computed_tokens = mm_end
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == {0}
# One position past the range end: the +1 look-ahead has now passed it.
request.num_computed_tokens = mm_end + 1
scheduler._free_encoder_inputs(request)
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before."""