[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
+5 -2
View File
@@ -640,7 +640,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator, "enable_adaptive_verification", False
),
attn_groups=self.attn_groups,
attn_cg_support=attn_cg_support,
req_states=self.req_states,
query_start_loc=self.input_buffers.query_start_loc,
num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step,
@@ -1646,12 +1645,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get batch descriptor and sync across DP ranks.
num_reqs = len(scheduler_output.num_scheduled_tokens)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
max_query_len: int | None = max(scheduler_output.num_scheduled_tokens.values())
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
scheduler_output, dummy_run
)
if batch_req_state is not None:
num_toks = batch_req_state.num_tokens
if batch_req_state.has_prefill:
# Varlen decode graphs replay decodes only, and their bound
# alone would admit a short prefill.
max_query_len = None
if self.pcp_manager is not None:
num_toks = self.pcp_manager.get_num_tokens_for_dispatch(
batch_req_state.num_scheduled_tokens,