[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-05 14:54:11 -04:00
committed by GitHub
parent c32495d58f
commit 74c5cbcd7b
15 changed files with 190 additions and 18 deletions
+6
View File
@@ -146,6 +146,9 @@ def run_mixed_prefill_decode_warmup(
)
decode_prefill_output = SchedulerOutput.make_empty()
decode_prefill_output.num_spec_tokens_to_schedule = (
model_runner.num_speculative_steps
)
decode_prefill_output.scheduled_new_reqs = [
NewRequestData(
req_id=decode_req_id,
@@ -175,6 +178,7 @@ def run_mixed_prefill_decode_warmup(
]
mixed_output = SchedulerOutput.make_empty()
mixed_output.num_spec_tokens_to_schedule = model_runner.num_speculative_steps
mixed_output.scheduled_cached_reqs = cached_decode_req
mixed_output.scheduled_new_reqs = [
NewRequestData(
@@ -353,6 +357,7 @@ def _warmup_kernels(
]
prefill_output = SchedulerOutput.make_empty()
prefill_output.num_spec_tokens_to_schedule = num_spec_steps
prefill_output.scheduled_new_reqs = new_reqs
prefill_output.num_scheduled_tokens = {rid: prompt_len for rid in req_ids}
prefill_output.total_num_scheduled_tokens = prompt_len * num_reqs
@@ -412,6 +417,7 @@ def _warmup_kernels(
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
decode_output = SchedulerOutput.make_empty()
decode_output.num_spec_tokens_to_schedule = num_spec_steps
decode_output.scheduled_cached_reqs = cached_req_data
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
decode_output.scheduled_spec_decode_tokens = step_spec_tokens