[Bugfix][Spec Decode] Skip uniform spec-decode padding for diffusion models (#47464)
Signed-off-by: kl527 <kl527@cornell.edu> Signed-off-by: Kyung Sub Lee (Daniel) <66861800+kl527@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
066f02ae94
commit
c5b66233b2
@@ -1356,6 +1356,41 @@ def test_spec_decode_padding_first_decode_step():
|
||||
assert out.scheduled_spec_decode_tokens[r2.request_id] == [-1] * num_spec
|
||||
|
||||
|
||||
def test_spec_decode_padding_skipped_for_diffusion():
|
||||
"""Diffusion spec tokens are the fixed-size denoising canvas, not
|
||||
rejectable drafts: a first-decode-step request must keep its 1-token span
|
||||
instead of being padded to 1 + num_spec_tokens, which would overflow the
|
||||
canvas.
|
||||
"""
|
||||
num_spec = 3
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=num_spec,
|
||||
enable_prefix_caching=True,
|
||||
block_size=16,
|
||||
)
|
||||
# Diffusion schedulers initialize this to 0 (model_config.is_diffusion).
|
||||
scheduler.num_sampled_tokens_per_step = 0
|
||||
r1, r2 = create_requests(
|
||||
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
|
||||
)
|
||||
|
||||
scheduler.add_request(r1)
|
||||
out = scheduler.schedule()
|
||||
assert out.num_scheduled_tokens[r1.request_id] == 33
|
||||
_model_output(scheduler, out, [[100]])
|
||||
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
|
||||
|
||||
# r2 arrives; its whole prompt is a prefix-cache hit -> needs exactly
|
||||
# 1 token while r1 is a running speculative decode.
|
||||
scheduler.add_request(r2)
|
||||
out = scheduler.schedule()
|
||||
|
||||
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
|
||||
# r2 keeps its true 1-token span; no placeholder drafts are attached.
|
||||
assert out.num_scheduled_tokens[r2.request_id] == 1
|
||||
assert r2.request_id not in out.scheduled_spec_decode_tokens
|
||||
|
||||
|
||||
def test_spec_decode_padding_skipped_with_prefill_in_batch():
|
||||
"""Padding is skipped when the batch contains a prefill chunk: the batch is
|
||||
already mixed/non-uniform, so padding a new decode request buys nothing.
|
||||
|
||||
@@ -811,8 +811,10 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Pad new decode requests to uniform spec decoding size to
|
||||
# preserve full cudagraph for this step.
|
||||
# Not for diffusion where draft tokens can't be padded.
|
||||
if (
|
||||
(self.num_spec_tokens > 0 and self.dynamic_sd_lookup is None)
|
||||
and self.num_sampled_tokens_per_step > 0
|
||||
and num_new_tokens == 1
|
||||
and (scheduled_running_reqs and not prefill_scheduled)
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user