[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+34 -3
View File
@@ -55,6 +55,7 @@ def _make_runner(
num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32)
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = decode_query_len
runner.adaptive_verification = None
runner.req_states = SimpleNamespace(
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
# The runner keeps this as min(num_computed_tokens, prefill_len).
@@ -73,6 +74,7 @@ def _uniform_token_count(
scheduler_output = SimpleNamespace(
num_scheduled_tokens={req_id: query_len for req_id in req_states},
total_num_scheduled_tokens=query_len * len(req_states),
scheduled_spec_decode_tokens={},
)
runner = _make_runner(req_states, decode_query_len=query_len)
_, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run)
@@ -85,6 +87,33 @@ def test_spec_decode_batch_is_uniform_decode():
assert _uniform_token_count(decodes, 8) == 8
def test_adaptive_verification_sizes_only_batches_with_drafts():
decodes = {"d0": (16, 16), "d1": (16, 16)}
runner = _make_runner(decodes, decode_query_len=8)
manager = SimpleNamespace(
get_num_tokens=lambda _num_tokens_per_req, _draft_tokens: 12
)
runner.adaptive_verification = manager
scheduler_output = SimpleNamespace(
num_scheduled_tokens={req_id: 8 for req_id in decodes},
total_num_scheduled_tokens=16,
scheduled_spec_decode_tokens={},
)
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert state is not None
assert state.num_tokens == 16
assert uniform_tok_count == 8
scheduler_output.scheduled_spec_decode_tokens = {
req_id: [1, 2] for req_id in decodes
}
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert state is not None
assert state.num_tokens == 12
assert uniform_tok_count is None
def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
# Two prompt chunks whose query length coincides with the K+1 spec-decode
# query length must reject the batch (issue #49918).
@@ -116,14 +145,14 @@ def test_dummy_batches_stay_uniform_decode():
def test_sort_batch_req_ids_no_spec():
# decode_query_len == 1: plain ascending order (decodes first).
num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1}
assert sort_batch_req_ids(num_tokens_per_req, 1) == ["d1", "d2", "p2", "p1"]
assert sort_batch_req_ids(num_tokens_per_req, {}, 1) == ["d1", "d2", "p2", "p1"]
def test_sort_batch_req_ids_spec_decode():
# decode_query_len == 2 (MTP k=1): uniform decodes lead, then the 1-token
# chunked-prefill tail, then longer prefills.
num_tokens_per_req = {"tail": 1, "d1": 2, "p1": 100, "d2": 2}
assert sort_batch_req_ids(num_tokens_per_req, 2) == ["d1", "d2", "tail", "p1"]
assert sort_batch_req_ids(num_tokens_per_req, {}, 2) == ["d1", "d2", "tail", "p1"]
def test_spec_decodes_lead_short_prefill_tail():
@@ -131,7 +160,7 @@ def test_spec_decodes_lead_short_prefill_tail():
# uniform 2-token decodes as decodes even when a 1-token prefill tail is
# in the batch (indexer-style: require_uniform, threshold=1+k).
num_tokens_per_req = {"tail": 1, **{f"d{i}": 2 for i in range(8)}}
req_ids = sort_batch_req_ids(num_tokens_per_req, 2)
req_ids = sort_batch_req_ids(num_tokens_per_req, {}, 2)
query_lens = [num_tokens_per_req[r] for r in req_ids]
assert query_lens == [2] * 8 + [1]
@@ -155,6 +184,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
"""
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = 8
runner.adaptive_verification = None
# State arrays in state-index order: a prefilling request, then two decodes.
runner.req_states = SimpleNamespace(
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
@@ -164,6 +194,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
scheduler_output = SimpleNamespace(
num_scheduled_tokens={"decode_a": 8, "decode_b": 8},
total_num_scheduled_tokens=16,
scheduled_spec_decode_tokens={},
)
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)