[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-05 14:54:11 -04:00
committed by GitHub
parent c32495d58f
commit 74c5cbcd7b
15 changed files with 190 additions and 18 deletions
+7 -1
View File
@@ -2057,6 +2057,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
cudagraph_stats=cudagraph_stats,
num_spec_tokens_to_schedule=scheduler_output.num_spec_tokens_to_schedule,
)
if not self.is_last_pp_rank:
@@ -2086,6 +2087,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
cudagraph_stats = self.execute_model_state.cudagraph_stats
num_spec_tokens = self.execute_model_state.num_spec_tokens_to_schedule
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -2237,9 +2239,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.next_prefill_tokens,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
num_speculative_tokens=num_spec_tokens,
dp_sync=dp_sync,
mm_inputs=mm_inputs,
)
if num_spec_tokens < self.num_speculative_steps:
draft_tokens[:, num_spec_tokens:] = -1
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
if self.adaptive_verification is not None:
self.adaptive_verification.record_confidences(
@@ -2251,7 +2256,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# not have a speculator (i.e. self.speculator is None)
self.draft_tokens_handler.set_draft_tokens(
input_batch,
self.req_states.draft_tokens[input_batch.idx_mapping],
self.req_states.draft_tokens[input_batch.idx_mapping, :num_spec_tokens],
)
if self.pp_handler is not None:
self.pp_handler.broadcast_drafts(
@@ -2396,6 +2401,7 @@ class ExecuteModelState(NamedTuple):
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
cudagraph_stats: CUDAGraphStat | None
num_spec_tokens_to_schedule: int
class BatchReqState(NamedTuple):