[Bugfix][Spec Decode] Fix EAGLE drafter multimodal encoder cache misses (#46315)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Nick Hill
2026-06-22 18:14:02 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Roger Wang
parent ac614587f5
commit 3e6529cc0e
7 changed files with 331 additions and 18 deletions
+21 -10
View File
@@ -68,15 +68,19 @@ class EncoderRunner:
query_start_loc: np.ndarray,
prefill_lens: np.ndarray,
computed_prefill_lens: np.ndarray,
draft_lookahead: int = 0,
) -> tuple[list[torch.Tensor], torch.Tensor]:
is_prefilling = (computed_prefill_lens < prefill_lens).tolist()
all_decode = not any(is_prefilling)
if all_decode:
if draft_lookahead:
computed_prefill_lens = computed_prefill_lens + draft_lookahead
is_prefilling_np = computed_prefill_lens < prefill_lens
if not is_prefilling_np.any():
# All decode requests, so no need to gather any embeddings.
return [], torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device=self.device
)
is_prefilling = is_prefilling_np.tolist()
query_start = computed_prefill_lens.tolist()
query_end = (computed_prefill_lens + num_scheduled_tokens).tolist()
@@ -89,11 +93,12 @@ class EncoderRunner:
# OPTIMIZATION: Skip decode requests.
continue
cur_query_start = query_start[i]
cur_query_end = query_end[i]
mm_features = self.encoder_cache.mm_features[req_id]
lo, hi = get_mm_features_in_window(
mm_features,
start=query_start[i],
end=query_end[i],
mm_features, start=cur_query_start, end=cur_query_end
)
for idx in range(lo, hi):
mm_feature = mm_features[idx]
@@ -101,8 +106,8 @@ class EncoderRunner:
start_pos = pos_info.offset
num_encoder_tokens = pos_info.length
start_idx = max(query_start[i] - start_pos, 0)
end_idx = min(query_end[i] - start_pos, num_encoder_tokens)
start_idx = max(cur_query_start - start_pos, 0)
end_idx = min(cur_query_end - start_pos, num_encoder_tokens)
assert start_idx < end_idx
curr_embeds_start, curr_embeds_end = (
pos_info.get_embeds_indices_in_range(start_idx, end_idx)
@@ -114,7 +119,13 @@ class EncoderRunner:
mm_hash = mm_feature.identifier
encoder_output = self.encoder_cache.encoder_outputs.get(mm_hash, None)
assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
if encoder_output is None:
# A feature starting at/after the processed boundary is only
# reached via the drafter's +1 look-ahead and might not be
# encoded yet; fall back to the token embedding for drafting.
if start_pos + draft_lookahead >= cur_query_end:
continue
raise RuntimeError(f"Encoder cache miss for {mm_hash}.")
if (is_embed := pos_info.is_embed) is not None:
is_embed = is_embed[start_idx:end_idx]
@@ -122,7 +133,7 @@ class EncoderRunner:
else:
mm_embeds_item = encoder_output[start_idx:end_idx]
req_start_pos = query_start_loc[i] + start_pos - query_start[i]
req_start_pos = query_start_loc[i] + start_pos - cur_query_start
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
True if is_embed is None else is_embed
)