[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Claude Opus 4.8
Roger Wang
parent
ac614587f5
commit
3e6529cc0e
@@ -4708,6 +4708,38 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (self.use_eagle).
|
||||
scheduler.use_eagle = True
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=250,
|
||||
mm_positions=mm_positions,
|
||||
)[0]
|
||||
manager = scheduler.encoder_cache_manager
|
||||
manager.allocate(request, 0)
|
||||
mm_end = 150 # offset + length
|
||||
|
||||
# Confirmed progress reaches the range end: without spec decode this frees
|
||||
# (see test below), but the drafter's +1 look-ahead still needs it.
|
||||
request.num_computed_tokens = mm_end
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == {0}
|
||||
|
||||
# One position past the range end: the +1 look-ahead has now passed it.
|
||||
request.num_computed_tokens = mm_end + 1
|
||||
scheduler._free_encoder_inputs(request)
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
|
||||
Reference in New Issue
Block a user