[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
@@ -0,0 +1,25 @@
model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark"
accuracy_threshold: 0.92
# lower acceptance length since trimming drafts at higher concurrency reduces
# acceptance length, this is very loose lower bound for smoke testing
min_acceptance_length: 1.2
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
server_args: >-
--tokenizer-mode deepseek_v4
--trust-remote-code
--max-model-len 8192
--tensor-parallel-size 4
--enable-expert-parallel
--block-size 256
--gpu-memory-utilization 0.5
--kv-cache-dtype fp8
--attention_config.use_fp4_indexer_cache=True
--max-num-batched-tokens 16384
--max-num-seqs 128
--speculative-config '{"method":"dspark",
"model":"deepseek-ai/DeepSeek-V4-Flash-DSpark",
"attention_backend":"FLASH_ATTN","num_speculative_tokens":7,
"draft_sample_method":"probabilistic",
"enable_adaptive_verification":true}'
@@ -0,0 +1 @@
DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
@@ -12,6 +12,7 @@ pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
import shlex
import pytest
import requests
import yaml
from tests.utils import RemoteOpenAIServer
@@ -65,6 +66,28 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
return results
def get_acceptance_length(server_url: str) -> float:
"""Mean tokens emitted per verification step, from the server's counters.
1.0 means every draft was rejected (speculation bought nothing); the
theoretical maximum is 1 + num_speculative_tokens.
"""
response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics")
response.raise_for_status()
counters: dict[str, float] = {}
for line in response.text.splitlines():
if line.startswith("vllm:spec_decode_num_"):
name, _, value = line.partition(" ")
counters[name.split("{")[0]] = float(value)
num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0)
num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0)
assert num_drafts > 0, (
"no drafts recorded; speculative decoding did not run for this config"
)
return 1.0 + num_accepted / num_drafts
def test_gsm8k_correctness(config_filename):
"""Test GSM8K correctness for a given model configuration."""
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
@@ -168,4 +191,16 @@ def test_gsm8k_correctness(config_filename):
f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
)
# Speculative configs additionally assert that drafts are actually
# landing: accuracy alone passes even when every draft is rejected.
min_acceptance_length = eval_config.get("min_acceptance_length")
if min_acceptance_length is not None:
acceptance_length = get_acceptance_length(server_url)
print(f" Mean acceptance length: {acceptance_length:.3f}")
print(f" Minimum acceptance length: {min_acceptance_length:.3f}")
assert acceptance_length >= min_acceptance_length, (
f"Acceptance length too low: {acceptance_length:.3f} < "
f"{min_acceptance_length:.3f}"
)
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
from vllm.v1.worker.gpu.async_utils import StepTimingSample
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
)
def make_manager(
confidences: np.ndarray, verify_cost_ms: np.ndarray
) -> AdaptiveVerificationManager:
num_reqs, num_steps = confidences.shape
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.num_speculative_steps = num_steps
manager._stale_confidences = [SimpleNamespace(np=confidences)]
manager._stale_idx = 0
manager.req_states = SimpleNamespace(
req_id_to_index={"low": 0, "high": 1},
num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32),
prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)),
)
manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms)
manager._max_total_logits = 1 << 30
manager.num_bonus_tokens = 1
return manager
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
# Verification is cheap up to two extra tokens, then jumps 100x; only the
# highest-confidence draft is worth the cheap slot.
manager = make_manager(
np.array([[0.1, 0.1], [0.9, 0.9]], dtype=np.float32),
np.array([1.0, 1.0, 1.0, 1.0, 100.0, 100.0, 100.0]),
)
manager.get_num_tokens(
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
assert draft_budget == 1
assert valid_drafts == {"low": 2, "high": 2}
assert num_non_draft_tokens == {"low": 1, "high": 1}
def test_profiled_batches_seed_cost_curves_via_consumer():
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
manager.num_speculative_steps = 7
manager.num_bonus_tokens = 1
curves: dict[str, list[tuple[int, float]]] = {}
manager.set_cost_curves = lambda draft, verify: curves.update(
draft=draft, verify=verify
)
timings = [
StepTimingSample(
forward_ms=float(batch["num_tokens"]),
drafter_ms=1.0,
num_target_tokens=batch["num_tokens"],
num_reqs=batch["num_tokens"] // 8,
# Only the captured sizes replay a graph; the tail sizes run eager.
full_cudagraph=batch["num_tokens"] <= 1024,
)
for batch in manager.batches_to_profile([8, 1024])
]
manager.set_initial_cost_curves(timings)
# Tail beyond the last capture size: 1.5x then doubling to the max.
assert curves["verify"] == [
(8, 8.0),
(1024, 1024.0),
(1536, 1536.0),
(2048, 2048.0),
(4096, 4096.0),
]
# Eager batches must not contribute to the draft curve: keyed by request
# count they would land inside the captured range and, once made monotonic,
# smear that eager cost across every larger request count.
assert curves["draft"] == [(1, 1.0), (128, 1.0)]
def test_compact_batch_preserves_totals_and_bounds():
# The CPU placeholder layout must keep the batch total equal to the GPU
# total and every verification row within decode_query_len, or downstream
# CPU metadata desyncs from the reallocated GPU boundaries.
manager = make_manager(
np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32),
np.array([1.0] * 44 + [100.0] * 3),
)
manager.req_states.req_id_to_index["prefill"] = 2
manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32)
manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32)
num_tokens = manager.get_num_tokens(
{"low": 3, "high": 3, "prefill": 40},
{"low": [1, 2], "high": [3, 4]},
)
scheduled = np.array([3, 3, 40], dtype=np.int32)
drafts = np.array([2, 2, 0], dtype=np.int32)
cu_num_logits_np = np.array([0, 3, 6, 7], dtype=np.int32)
compacted, _ = manager.compact_batch(drafts, scheduled, cu_num_logits_np)
assert int(compacted.sum()) == num_tokens
num_steps = manager.num_speculative_steps
assert (compacted[:2] <= 1 + num_steps).all()
assert compacted[2] == 40
def test_budget_caps_at_one_rejection_sampler_chunk():
# The chunked verification path cannot address the compacted logits
# layout, so the budget must keep total logits within a single chunk.
manager = make_manager(
np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32),
np.ones(7),
)
manager._max_total_logits = 3 # 2 bonus logits + at most 1 draft
manager.get_num_tokens(
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
_, _, draft_budget = manager._batch_budget
assert draft_budget <= 1
def test_zero_budget_rebuilds_cpu_cu_num_logits():
# When one bonus row per request already overflows a verification chunk, the
# budget clamps to zero but the batch still needs chunking. Every capacity is
# zeroed on device, so the CPU can name that layout exactly -- and must, since
# _iter_request_chunks slices the compacted logits with these offsets.
#
# The third request is a chunked prefill (no drafts, still mid-prompt). The
# runner gives *every* request num_bonus_tokens logits rows regardless
# (num_logits = num_draft_tokens_per_req + num_bonus_tokens), so the rebuilt
# offsets stay uniform rather than skipping non-verification rows.
manager = make_manager(
np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32),
np.ones(64),
)
manager.req_states.req_id_to_index["prefill"] = 2
manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32)
manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32)
manager._max_total_logits = 2 # < 3 requests * 1 bonus token
manager.get_num_tokens(
{"low": 3, "high": 3, "prefill": 40},
{"low": [1, 2], "high": [3, 4]},
)
_, _, draft_budget = manager._batch_budget
assert draft_budget == 0
scheduled = np.array([3, 3, 40], dtype=np.int32)
drafts = np.array([2, 2, 0], dtype=np.int32)
scheduled_cu_num_logits = np.array([0, 3, 6, 7], dtype=np.int32)
compacted, cu_num_logits_np = manager.compact_batch(
drafts, scheduled, scheduled_cu_num_logits
)
# One bonus row per request, matching cumsum(capacities + num_bonus_tokens)
# with every capacity zeroed -- the prefill row included.
expected = np.arange(4, dtype=np.int32) * manager.num_bonus_tokens
assert np.array_equal(cu_num_logits_np, expected)
assert cu_num_logits_np.dtype == scheduled_cu_num_logits.dtype
# The prefill keeps its scheduled tokens; only drafts are dropped.
assert np.array_equal(compacted, np.array([1, 1, 40], dtype=np.int32))
@@ -42,6 +42,7 @@ def mock_model_runner_with_req_states():
runner.prompt_logprobs_worker = None
runner.is_last_pp_rank = False
runner.pooling_runner = None
runner.adaptive_verification = None
# Mock staged writes — they use Triton kernels that require GPU
runner.req_states.apply_staged_writes = Mock()
+34 -3
View File
@@ -55,6 +55,7 @@ def _make_runner(
num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32)
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = decode_query_len
runner.adaptive_verification = None
runner.req_states = SimpleNamespace(
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
# The runner keeps this as min(num_computed_tokens, prefill_len).
@@ -73,6 +74,7 @@ def _uniform_token_count(
scheduler_output = SimpleNamespace(
num_scheduled_tokens={req_id: query_len for req_id in req_states},
total_num_scheduled_tokens=query_len * len(req_states),
scheduled_spec_decode_tokens={},
)
runner = _make_runner(req_states, decode_query_len=query_len)
_, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run)
@@ -85,6 +87,33 @@ def test_spec_decode_batch_is_uniform_decode():
assert _uniform_token_count(decodes, 8) == 8
def test_adaptive_verification_sizes_only_batches_with_drafts():
decodes = {"d0": (16, 16), "d1": (16, 16)}
runner = _make_runner(decodes, decode_query_len=8)
manager = SimpleNamespace(
get_num_tokens=lambda _num_tokens_per_req, _draft_tokens: 12
)
runner.adaptive_verification = manager
scheduler_output = SimpleNamespace(
num_scheduled_tokens={req_id: 8 for req_id in decodes},
total_num_scheduled_tokens=16,
scheduled_spec_decode_tokens={},
)
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert state is not None
assert state.num_tokens == 16
assert uniform_tok_count == 8
scheduler_output.scheduled_spec_decode_tokens = {
req_id: [1, 2] for req_id in decodes
}
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert state is not None
assert state.num_tokens == 12
assert uniform_tok_count is None
def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
# Two prompt chunks whose query length coincides with the K+1 spec-decode
# query length must reject the batch (issue #49918).
@@ -116,14 +145,14 @@ def test_dummy_batches_stay_uniform_decode():
def test_sort_batch_req_ids_no_spec():
# decode_query_len == 1: plain ascending order (decodes first).
num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1}
assert sort_batch_req_ids(num_tokens_per_req, 1) == ["d1", "d2", "p2", "p1"]
assert sort_batch_req_ids(num_tokens_per_req, {}, 1) == ["d1", "d2", "p2", "p1"]
def test_sort_batch_req_ids_spec_decode():
# decode_query_len == 2 (MTP k=1): uniform decodes lead, then the 1-token
# chunked-prefill tail, then longer prefills.
num_tokens_per_req = {"tail": 1, "d1": 2, "p1": 100, "d2": 2}
assert sort_batch_req_ids(num_tokens_per_req, 2) == ["d1", "d2", "tail", "p1"]
assert sort_batch_req_ids(num_tokens_per_req, {}, 2) == ["d1", "d2", "tail", "p1"]
def test_spec_decodes_lead_short_prefill_tail():
@@ -131,7 +160,7 @@ def test_spec_decodes_lead_short_prefill_tail():
# uniform 2-token decodes as decodes even when a 1-token prefill tail is
# in the batch (indexer-style: require_uniform, threshold=1+k).
num_tokens_per_req = {"tail": 1, **{f"d{i}": 2 for i in range(8)}}
req_ids = sort_batch_req_ids(num_tokens_per_req, 2)
req_ids = sort_batch_req_ids(num_tokens_per_req, {}, 2)
query_lens = [num_tokens_per_req[r] for r in req_ids]
assert query_lens == [2] * 8 + [1]
@@ -155,6 +184,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
"""
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = 8
runner.adaptive_verification = None
# State arrays in state-index order: a prefilling request, then two decodes.
runner.req_states = SimpleNamespace(
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
@@ -164,6 +194,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
scheduler_output = SimpleNamespace(
num_scheduled_tokens={"decode_a": 8, "decode_b": 8},
total_num_scheduled_tokens=16,
scheduled_spec_decode_tokens={},
)
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)