[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
@@ -0,0 +1,25 @@
model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark"
accuracy_threshold: 0.92
# lower acceptance length since trimming drafts at higher concurrency reduces
# acceptance length, this is very loose lower bound for smoke testing
min_acceptance_length: 1.2
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
server_args: >-
--tokenizer-mode deepseek_v4
--trust-remote-code
--max-model-len 8192
--tensor-parallel-size 4
--enable-expert-parallel
--block-size 256
--gpu-memory-utilization 0.5
--kv-cache-dtype fp8
--attention_config.use_fp4_indexer_cache=True
--max-num-batched-tokens 16384
--max-num-seqs 128
--speculative-config '{"method":"dspark",
"model":"deepseek-ai/DeepSeek-V4-Flash-DSpark",
"attention_backend":"FLASH_ATTN","num_speculative_tokens":7,
"draft_sample_method":"probabilistic",
"enable_adaptive_verification":true}'
@@ -0,0 +1 @@
DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
@@ -12,6 +12,7 @@ pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
import shlex
import pytest
import requests
import yaml
from tests.utils import RemoteOpenAIServer
@@ -65,6 +66,28 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
return results
def get_acceptance_length(server_url: str) -> float:
"""Mean tokens emitted per verification step, from the server's counters.
1.0 means every draft was rejected (speculation bought nothing); the
theoretical maximum is 1 + num_speculative_tokens.
"""
response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics")
response.raise_for_status()
counters: dict[str, float] = {}
for line in response.text.splitlines():
if line.startswith("vllm:spec_decode_num_"):
name, _, value = line.partition(" ")
counters[name.split("{")[0]] = float(value)
num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0)
num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0)
assert num_drafts > 0, (
"no drafts recorded; speculative decoding did not run for this config"
)
return 1.0 + num_accepted / num_drafts
def test_gsm8k_correctness(config_filename):
"""Test GSM8K correctness for a given model configuration."""
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
@@ -168,4 +191,16 @@ def test_gsm8k_correctness(config_filename):
f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
)
# Speculative configs additionally assert that drafts are actually
# landing: accuracy alone passes even when every draft is rejected.
min_acceptance_length = eval_config.get("min_acceptance_length")
if min_acceptance_length is not None:
acceptance_length = get_acceptance_length(server_url)
print(f" Mean acceptance length: {acceptance_length:.3f}")
print(f" Minimum acceptance length: {min_acceptance_length:.3f}")
assert acceptance_length >= min_acceptance_length, (
f"Acceptance length too low: {acceptance_length:.3f} < "
f"{min_acceptance_length:.3f}"
)
print(f"✅ GSM8K test passed for {eval_config['model_name']}")