[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement (#57053)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -146,6 +146,9 @@ def run_mixed_prefill_decode_warmup(
|
||||
)
|
||||
|
||||
decode_prefill_output = SchedulerOutput.make_empty()
|
||||
decode_prefill_output.num_spec_tokens_to_schedule = (
|
||||
model_runner.num_speculative_steps
|
||||
)
|
||||
decode_prefill_output.scheduled_new_reqs = [
|
||||
NewRequestData(
|
||||
req_id=decode_req_id,
|
||||
@@ -175,6 +178,7 @@ def run_mixed_prefill_decode_warmup(
|
||||
]
|
||||
|
||||
mixed_output = SchedulerOutput.make_empty()
|
||||
mixed_output.num_spec_tokens_to_schedule = model_runner.num_speculative_steps
|
||||
mixed_output.scheduled_cached_reqs = cached_decode_req
|
||||
mixed_output.scheduled_new_reqs = [
|
||||
NewRequestData(
|
||||
@@ -353,6 +357,7 @@ def _warmup_kernels(
|
||||
]
|
||||
|
||||
prefill_output = SchedulerOutput.make_empty()
|
||||
prefill_output.num_spec_tokens_to_schedule = num_spec_steps
|
||||
prefill_output.scheduled_new_reqs = new_reqs
|
||||
prefill_output.num_scheduled_tokens = {rid: prompt_len for rid in req_ids}
|
||||
prefill_output.total_num_scheduled_tokens = prompt_len * num_reqs
|
||||
@@ -412,6 +417,7 @@ def _warmup_kernels(
|
||||
step_spec_tokens[req_ids[i]] = [0] * num_spec_steps
|
||||
|
||||
decode_output = SchedulerOutput.make_empty()
|
||||
decode_output.num_spec_tokens_to_schedule = num_spec_steps
|
||||
decode_output.scheduled_cached_reqs = cached_req_data
|
||||
decode_output.num_scheduled_tokens = step_num_scheduled_tokens
|
||||
decode_output.scheduled_spec_decode_tokens = step_spec_tokens
|
||||
|
||||
Reference in New Issue
Block a user