Fix DFlash prefix cache corruption due to missing lookahead block (#42971)

Signed-off-by: Shreyas Kulkarni <shreyas.gp269@gmail.com>
This commit is contained in:
Shreyas Kulkarni
2026-06-02 12:06:33 +00:00
committed by GitHub
parent f69ede495b
commit 0eeba5eec1
2 changed files with 171 additions and 3 deletions
+17 -3
View File
@@ -4362,6 +4362,21 @@ class GPUModelRunner(
return None
def _input_fits_in_drafter(
self, common_attn_metadata: CommonAttentionMetadata | None
) -> bool:
if common_attn_metadata is None:
return False
assert self.speculative_config is not None
# DFlash queries one extra token (the bonus token) beyond num_spec_tokens
num_drafter_query_tokens = self.num_spec_tokens + (
1 if self.speculative_config.use_dflash() else 0
)
return (
common_attn_metadata.max_seq_len + num_drafter_query_tokens
<= self.effective_drafter_max_model_len
)
@torch.inference_mode
def sample_tokens(
self, grammar_output: "GrammarOutput | None"
@@ -4441,9 +4456,8 @@ class GPUModelRunner(
propose_drafts_after_bookkeeping = False
if spec_config is not None:
# Decide whether to run the drafter or zero out draft tokens.
input_fits_in_drafter = spec_decode_common_attn_metadata is not None and (
spec_decode_common_attn_metadata.max_seq_len + self.num_spec_tokens
<= self.effective_drafter_max_model_len
input_fits_in_drafter = self._input_fits_in_drafter(
spec_decode_common_attn_metadata
)
use_gpu_toks = (
spec_config.use_eagle()