[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
@@ -1412,6 +1412,67 @@ def test_dflash_acceptance_rates(
cleanup_dist_env_and_memory()
@pytest.fixture
def dspark_config():
target_model = "Qwen/Qwen3-4B-FP8"
draft_model = "deepseek-ai/dspark_qwen3_4b_block7"
return dict(
model=target_model,
trust_remote_code=True,
speculative_config={
"method": "dspark",
"model": draft_model,
"num_speculative_tokens": 7,
"attention_backend": "FLASH_ATTN",
"draft_sample_method": "probabilistic",
},
max_model_len=4096,
disable_log_stats=False,
)
@single_gpu_only
@large_gpu_mark(min_gb=24)
def test_dspark_correctness_and_acceptance_rate(dspark_config):
"""
E2E test for DSpark speculative decoding: acceptance rate/length
regression coverage plus GSM8K correctness, at temperature=1.0 to
exercise the probabilistic draft-sampling/rejection-sampling path
(not just greedy).
Uses Qwen/Qwen3-4B-FP8 as target with the dspark_qwen3_4b_block7 draft
model. Reference: measured over 12 runs of the full GSM8K set at
temperature=1.0 (prefix caching disabled to avoid cross-run reuse):
accuracy: min=0.782 max=0.814 mean=0.801
acceptance_rate: min=0.418 max=0.434 mean=0.428
acceptance_len: min=3.928 max=4.037 mean=3.994
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
"""
spec_llm = LLM(**dspark_config)
results = evaluate_gsm8k_offline(spec_llm, temperature=1.0)
gsm8k_accuracy = results["accuracy"]
metrics = spec_llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
acceptance_len = compute_acceptance_len(metrics)
print(
f"DSpark acceptance_rate={acceptance_rate:.2f}, "
f"acceptance_len={acceptance_len:.2f}, "
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
)
assert acceptance_rate >= 0.428 * 0.9
assert acceptance_len >= 3.994 * 0.9
assert gsm8k_accuracy >= 0.801 * 0.9
del spec_llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@single_gpu_only
def test_synthetic_acceptance_rate():
"""Verify that synthetic rejection sampling produces an acceptance