[Core] Avoid mixed length specdec batches via padding (#45237)

Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Zijing Liu <liuzijing2014@gmail.com>
This commit is contained in:
Yiliu Dong
2026-06-25 08:34:44 -07:00
committed by GitHub
co-authored by Nick Hill Jade Zheng Giancarlo Delfin Zijing Liu
parent 1744adc256
commit d490b98162
2 changed files with 120 additions and 0 deletions
+95
View File
@@ -1302,6 +1302,101 @@ def test_no_spec_tokens_scheduled_for_prefill_chunks():
assert len(output.scheduled_spec_decode_tokens[req.request_id]) == num_spec_tokens
def _model_output(scheduler, output, sampled):
"""Feed `sampled` (per-request list) back to the scheduler."""
req_ids = list(output.num_scheduled_tokens.keys())
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index={r: i for i, r in enumerate(req_ids)},
sampled_token_ids=sampled,
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
def test_spec_decode_padding_first_decode_step():
"""A request taking its first decode step (whole prompt already computed via
a prefix-cache hit) is padded with placeholder (-1) spec tokens so it enters
the worker with the same 1 + num_spec_tokens shape as the other speculative
decodes, keeping the batch uniform.
"""
num_spec = 3
scheduler = create_scheduler(
num_speculative_tokens=num_spec,
enable_prefix_caching=True,
block_size=16,
)
# Two identical 33-token prompts: 2 full blocks (32 tokens) get cached, so a
# second identical request hits num_computed == num_prompt_tokens - 1.
r1, r2 = create_requests(
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
)
# Drive r1 through prefill so its prompt blocks are cached, then give it real
# drafts so it is a running speculative decode (1 + num_spec shape).
scheduler.add_request(r1)
out = scheduler.schedule()
assert out.num_scheduled_tokens[r1.request_id] == 33
_model_output(scheduler, out, [[100]])
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
# r2 arrives; its whole prompt is a prefix-cache hit -> first decode step.
scheduler.add_request(r2)
out = scheduler.schedule()
# r1 verifies its real drafts.
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
# r2 is padded to the 1 + num_spec shape with placeholder (-1) drafts.
assert out.num_scheduled_tokens[r2.request_id] == 1 + num_spec
assert out.scheduled_spec_decode_tokens[r2.request_id] == [-1] * num_spec
def test_spec_decode_padding_skipped_with_prefill_in_batch():
"""Padding is skipped when the batch contains a prefill chunk: the batch is
already mixed/non-uniform, so padding a new decode request buys nothing.
"""
num_spec = 3
scheduler = create_scheduler(
num_speculative_tokens=num_spec,
enable_prefix_caching=True,
block_size=16,
max_num_batched_tokens=64,
)
# r_warm + r_candidate share a prompt so r_candidate gets a full prefix hit.
r_warm, r_candidate = create_requests(
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=1
)
# r_long has a different, long prompt that prefills over multiple chunks.
(r_long,) = create_requests(num_requests=1, num_tokens=100, max_tokens=16)
# Warm the prefix cache with r_warm's prompt (it finishes; blocks stay cached).
scheduler.add_request(r_warm)
out = scheduler.schedule()
assert out.num_scheduled_tokens[r_warm.request_id] == 33
_model_output(scheduler, out, [[100]])
assert r_warm.request_id in scheduler.finished_req_ids
# Start r_long; after one chunk it remains a prefill chunk in the running queue.
scheduler.add_request(r_long)
out = scheduler.schedule()
_model_output(scheduler, out, [[]]) # still prefilling, no sampled token
assert r_long.is_prefill_chunk
# r_candidate arrives (prefix-cache hit -> first decode step) alongside the
# in-flight prefill chunk.
scheduler.add_request(r_candidate)
out = scheduler.schedule()
# The batch has a prefill chunk, so r_candidate is NOT padded.
assert r_long.request_id in out.num_scheduled_tokens
assert out.num_scheduled_tokens[r_candidate.request_id] == 1
assert r_candidate.request_id not in out.scheduled_spec_decode_tokens
def test_scheduler_stats_waiting_queues():
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
# Create scheduler with limited capacity so we can have waiting requests
+25
View File
@@ -415,6 +415,8 @@ class Scheduler(SchedulerInterface):
encoder_compute_budget = self.max_num_encoder_input_tokens
# Spec decode-related.
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
# Whether the running batch contains any prefill requests.
prefill_scheduled = False
# For logging.
scheduled_timestamp = time.monotonic()
@@ -573,6 +575,7 @@ class Scheduler(SchedulerInterface):
# Schedule the request.
scheduled_running_reqs.append(request)
prefill_scheduled |= request.is_prefill_chunk
request_id = request.request_id
req_to_new_blocks[request_id] = new_blocks
num_scheduled_tokens[request_id] = num_new_tokens
@@ -778,6 +781,7 @@ class Scheduler(SchedulerInterface):
encoder_inputs_to_schedule = None
external_load_encoder_input = []
new_encoder_compute_budget = encoder_compute_budget
pad_spec_decode = False
if load_kv_async:
# KVTransfer: loading remote KV, do not allocate for new work.
@@ -793,6 +797,23 @@ class Scheduler(SchedulerInterface):
# `request.num_prompt_tokens` to consider the resumed
# requests, which have output tokens.
num_new_tokens = request.num_tokens - num_computed_tokens
# Pad new decode requests to uniform spec decoding size to
# preserve full cudagraph for this step.
if (
(self.num_spec_tokens > 0 and self.dynamic_sd_lookup is None)
and num_new_tokens == 1
and (scheduled_running_reqs and not prefill_scheduled)
):
num_new_tokens = 1 + self.num_spec_tokens
if (
num_new_tokens > token_budget
or num_computed_tokens + num_new_tokens > self.max_model_len
):
# Prefer to not schedule than schedule un-padded here.
break
pad_spec_decode = True
threshold = self.scheduler_config.long_prefill_token_threshold
if 0 < threshold < num_new_tokens:
num_new_tokens = threshold
@@ -957,6 +978,10 @@ class Scheduler(SchedulerInterface):
token_budget -= num_new_tokens
request.status = RequestStatus.RUNNING
request.num_computed_tokens = num_computed_tokens
if pad_spec_decode:
scheduled_spec_decode_tokens[request_id] = [
-1
] * self.num_spec_tokens
# Only track requests that will still be prefilling after this chunk.
if num_computed_tokens + num_new_tokens < request.num_tokens:
self._inflight_prefills.add(request)