[Bugfix][Spec Decode] Skip uniform spec-decode padding for diffusion models (#47464)

Signed-off-by: kl527 <kl527@cornell.edu>
Signed-off-by: Kyung Sub Lee (Daniel) <66861800+kl527@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kyung Sub Lee (Daniel)
2026-07-07 09:25:12 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent 066f02ae94
commit c5b66233b2
2 changed files with 37 additions and 0 deletions
+35
View File
@@ -1356,6 +1356,41 @@ def test_spec_decode_padding_first_decode_step():
assert out.scheduled_spec_decode_tokens[r2.request_id] == [-1] * num_spec
def test_spec_decode_padding_skipped_for_diffusion():
"""Diffusion spec tokens are the fixed-size denoising canvas, not
rejectable drafts: a first-decode-step request must keep its 1-token span
instead of being padded to 1 + num_spec_tokens, which would overflow the
canvas.
"""
num_spec = 3
scheduler = create_scheduler(
num_speculative_tokens=num_spec,
enable_prefix_caching=True,
block_size=16,
)
# Diffusion schedulers initialize this to 0 (model_config.is_diffusion).
scheduler.num_sampled_tokens_per_step = 0
r1, r2 = create_requests(
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
)
scheduler.add_request(r1)
out = scheduler.schedule()
assert out.num_scheduled_tokens[r1.request_id] == 33
_model_output(scheduler, out, [[100]])
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
# r2 arrives; its whole prompt is a prefix-cache hit -> needs exactly
# 1 token while r1 is a running speculative decode.
scheduler.add_request(r2)
out = scheduler.schedule()
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
# r2 keeps its true 1-token span; no placeholder drafts are attached.
assert out.num_scheduled_tokens[r2.request_id] == 1
assert r2.request_id not in out.scheduled_spec_decode_tokens
def test_spec_decode_padding_skipped_with_prefill_in_batch():
"""Padding is skipped when the batch contains a prefill chunk: the batch is
already mixed/non-uniform, so padding a new decode request buys nothing.
+2
View File
@@ -811,8 +811,10 @@ class Scheduler(SchedulerInterface):
# Pad new decode requests to uniform spec decoding size to
# preserve full cudagraph for this step.
# Not for diffusion where draft tokens can't be padded.
if (
(self.num_spec_tokens > 0 and self.dynamic_sd_lookup is None)
and self.num_sampled_tokens_per_step > 0
and num_new_tokens == 1
and (scheduled_running_reqs and not prefill_scheduled)
):