[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
Nick Hill
parent
3b4566c5cf
commit
a4eb3f25d6
@@ -640,7 +640,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator, "enable_adaptive_verification", False
|
||||
),
|
||||
attn_groups=self.attn_groups,
|
||||
attn_cg_support=attn_cg_support,
|
||||
req_states=self.req_states,
|
||||
query_start_loc=self.input_buffers.query_start_loc,
|
||||
num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step,
|
||||
@@ -1646,12 +1645,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get batch descriptor and sync across DP ranks.
|
||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||
num_toks = scheduler_output.total_num_scheduled_tokens
|
||||
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
|
||||
max_query_len: int | None = max(scheduler_output.num_scheduled_tokens.values())
|
||||
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
|
||||
scheduler_output, dummy_run
|
||||
)
|
||||
if batch_req_state is not None:
|
||||
num_toks = batch_req_state.num_tokens
|
||||
if batch_req_state.has_prefill:
|
||||
# Varlen decode graphs replay decodes only, and their bound
|
||||
# alone would admit a short prefill.
|
||||
max_query_len = None
|
||||
if self.pcp_manager is not None:
|
||||
num_toks = self.pcp_manager.get_num_tokens_for_dispatch(
|
||||
batch_req_state.num_scheduled_tokens,
|
||||
|
||||
Reference in New Issue
Block a user