[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -55,6 +55,7 @@ def _make_runner(
|
||||
num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32)
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = decode_query_len
|
||||
runner.adaptive_verification = None
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
|
||||
# The runner keeps this as min(num_computed_tokens, prefill_len).
|
||||
@@ -73,6 +74,7 @@ def _uniform_token_count(
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={req_id: query_len for req_id in req_states},
|
||||
total_num_scheduled_tokens=query_len * len(req_states),
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
runner = _make_runner(req_states, decode_query_len=query_len)
|
||||
_, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run)
|
||||
@@ -85,6 +87,33 @@ def test_spec_decode_batch_is_uniform_decode():
|
||||
assert _uniform_token_count(decodes, 8) == 8
|
||||
|
||||
|
||||
def test_adaptive_verification_sizes_only_batches_with_drafts():
|
||||
decodes = {"d0": (16, 16), "d1": (16, 16)}
|
||||
runner = _make_runner(decodes, decode_query_len=8)
|
||||
manager = SimpleNamespace(
|
||||
get_num_tokens=lambda _num_tokens_per_req, _draft_tokens: 12
|
||||
)
|
||||
runner.adaptive_verification = manager
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={req_id: 8 for req_id in decodes},
|
||||
total_num_scheduled_tokens=16,
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert state is not None
|
||||
assert state.num_tokens == 16
|
||||
assert uniform_tok_count == 8
|
||||
|
||||
scheduler_output.scheduled_spec_decode_tokens = {
|
||||
req_id: [1, 2] for req_id in decodes
|
||||
}
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert state is not None
|
||||
assert state.num_tokens == 12
|
||||
assert uniform_tok_count is None
|
||||
|
||||
|
||||
def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
|
||||
# Two prompt chunks whose query length coincides with the K+1 spec-decode
|
||||
# query length must reject the batch (issue #49918).
|
||||
@@ -116,14 +145,14 @@ def test_dummy_batches_stay_uniform_decode():
|
||||
def test_sort_batch_req_ids_no_spec():
|
||||
# decode_query_len == 1: plain ascending order (decodes first).
|
||||
num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1}
|
||||
assert sort_batch_req_ids(num_tokens_per_req, 1) == ["d1", "d2", "p2", "p1"]
|
||||
assert sort_batch_req_ids(num_tokens_per_req, {}, 1) == ["d1", "d2", "p2", "p1"]
|
||||
|
||||
|
||||
def test_sort_batch_req_ids_spec_decode():
|
||||
# decode_query_len == 2 (MTP k=1): uniform decodes lead, then the 1-token
|
||||
# chunked-prefill tail, then longer prefills.
|
||||
num_tokens_per_req = {"tail": 1, "d1": 2, "p1": 100, "d2": 2}
|
||||
assert sort_batch_req_ids(num_tokens_per_req, 2) == ["d1", "d2", "tail", "p1"]
|
||||
assert sort_batch_req_ids(num_tokens_per_req, {}, 2) == ["d1", "d2", "tail", "p1"]
|
||||
|
||||
|
||||
def test_spec_decodes_lead_short_prefill_tail():
|
||||
@@ -131,7 +160,7 @@ def test_spec_decodes_lead_short_prefill_tail():
|
||||
# uniform 2-token decodes as decodes even when a 1-token prefill tail is
|
||||
# in the batch (indexer-style: require_uniform, threshold=1+k).
|
||||
num_tokens_per_req = {"tail": 1, **{f"d{i}": 2 for i in range(8)}}
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, 2)
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, {}, 2)
|
||||
query_lens = [num_tokens_per_req[r] for r in req_ids]
|
||||
assert query_lens == [2] * 8 + [1]
|
||||
|
||||
@@ -155,6 +184,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
"""
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = 8
|
||||
runner.adaptive_verification = None
|
||||
# State arrays in state-index order: a prefilling request, then two decodes.
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
|
||||
@@ -164,6 +194,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={"decode_a": 8, "decode_b": 8},
|
||||
total_num_scheduled_tokens=16,
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
|
||||
Reference in New Issue
Block a user