[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -0,0 +1,25 @@
|
||||
model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark"
|
||||
accuracy_threshold: 0.92
|
||||
# lower acceptance length since trimming drafts at higher concurrency reduces
|
||||
# acceptance length, this is very loose lower bound for smoke testing
|
||||
min_acceptance_length: 1.2
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1800
|
||||
server_args: >-
|
||||
--tokenizer-mode deepseek_v4
|
||||
--trust-remote-code
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--block-size 256
|
||||
--gpu-memory-utilization 0.5
|
||||
--kv-cache-dtype fp8
|
||||
--attention_config.use_fp4_indexer_cache=True
|
||||
--max-num-batched-tokens 16384
|
||||
--max-num-seqs 128
|
||||
--speculative-config '{"method":"dspark",
|
||||
"model":"deepseek-ai/DeepSeek-V4-Flash-DSpark",
|
||||
"attention_backend":"FLASH_ATTN","num_speculative_tokens":7,
|
||||
"draft_sample_method":"probabilistic",
|
||||
"enable_adaptive_verification":true}'
|
||||
@@ -0,0 +1 @@
|
||||
DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
|
||||
@@ -12,6 +12,7 @@ pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
|
||||
import shlex
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
@@ -65,6 +66,28 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
return results
|
||||
|
||||
|
||||
def get_acceptance_length(server_url: str) -> float:
|
||||
"""Mean tokens emitted per verification step, from the server's counters.
|
||||
|
||||
1.0 means every draft was rejected (speculation bought nothing); the
|
||||
theoretical maximum is 1 + num_speculative_tokens.
|
||||
"""
|
||||
response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics")
|
||||
response.raise_for_status()
|
||||
counters: dict[str, float] = {}
|
||||
for line in response.text.splitlines():
|
||||
if line.startswith("vllm:spec_decode_num_"):
|
||||
name, _, value = line.partition(" ")
|
||||
counters[name.split("{")[0]] = float(value)
|
||||
|
||||
num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0)
|
||||
num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0)
|
||||
assert num_drafts > 0, (
|
||||
"no drafts recorded; speculative decoding did not run for this config"
|
||||
)
|
||||
return 1.0 + num_accepted / num_drafts
|
||||
|
||||
|
||||
def test_gsm8k_correctness(config_filename):
|
||||
"""Test GSM8K correctness for a given model configuration."""
|
||||
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
|
||||
@@ -168,4 +191,16 @@ def test_gsm8k_correctness(config_filename):
|
||||
f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
|
||||
)
|
||||
|
||||
# Speculative configs additionally assert that drafts are actually
|
||||
# landing: accuracy alone passes even when every draft is rejected.
|
||||
min_acceptance_length = eval_config.get("min_acceptance_length")
|
||||
if min_acceptance_length is not None:
|
||||
acceptance_length = get_acceptance_length(server_url)
|
||||
print(f" Mean acceptance length: {acceptance_length:.3f}")
|
||||
print(f" Minimum acceptance length: {min_acceptance_length:.3f}")
|
||||
assert acceptance_length >= min_acceptance_length, (
|
||||
f"Acceptance length too low: {acceptance_length:.3f} < "
|
||||
f"{min_acceptance_length:.3f}"
|
||||
)
|
||||
|
||||
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
|
||||
|
||||
Reference in New Issue
Block a user