[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -1412,6 +1412,67 @@ def test_dflash_acceptance_rates(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_config():
|
||||
target_model = "Qwen/Qwen3-4B-FP8"
|
||||
draft_model = "deepseek-ai/dspark_qwen3_4b_block7"
|
||||
|
||||
return dict(
|
||||
model=target_model,
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dspark",
|
||||
"model": draft_model,
|
||||
"num_speculative_tokens": 7,
|
||||
"attention_backend": "FLASH_ATTN",
|
||||
"draft_sample_method": "probabilistic",
|
||||
},
|
||||
max_model_len=4096,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_dspark_correctness_and_acceptance_rate(dspark_config):
|
||||
"""
|
||||
E2E test for DSpark speculative decoding: acceptance rate/length
|
||||
regression coverage plus GSM8K correctness, at temperature=1.0 to
|
||||
exercise the probabilistic draft-sampling/rejection-sampling path
|
||||
(not just greedy).
|
||||
|
||||
Uses Qwen/Qwen3-4B-FP8 as target with the dspark_qwen3_4b_block7 draft
|
||||
model. Reference: measured over 12 runs of the full GSM8K set at
|
||||
temperature=1.0 (prefix caching disabled to avoid cross-run reuse):
|
||||
accuracy: min=0.782 max=0.814 mean=0.801
|
||||
acceptance_rate: min=0.418 max=0.434 mean=0.428
|
||||
acceptance_len: min=3.928 max=4.037 mean=3.994
|
||||
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
|
||||
"""
|
||||
spec_llm = LLM(**dspark_config)
|
||||
|
||||
results = evaluate_gsm8k_offline(spec_llm, temperature=1.0)
|
||||
gsm8k_accuracy = results["accuracy"]
|
||||
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
acceptance_len = compute_acceptance_len(metrics)
|
||||
|
||||
print(
|
||||
f"DSpark acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
|
||||
)
|
||||
|
||||
assert acceptance_rate >= 0.428 * 0.9
|
||||
assert acceptance_len >= 3.994 * 0.9
|
||||
assert gsm8k_accuracy >= 0.801 * 0.9
|
||||
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_synthetic_acceptance_rate():
|
||||
"""Verify that synthetic rejection sampling produces an acceptance
|
||||
|
||||
Reference in New Issue
Block a user