[CI] Organize speculative decoding E2E tests by coverage (#50330)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b40d859c7b
commit
fcdc7c2e9c
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ...utils import compute_acceptance_len
|
||||
from ..utils import run_acceptance_length_eval
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DFlashCorrectnessConfig:
|
||||
model: str
|
||||
draft_model: str
|
||||
expected_accuracy: float
|
||||
expected_acceptance_len: float
|
||||
num_speculative_tokens: int = 16
|
||||
max_model_len: int = 4096
|
||||
max_num_seqs: int = 128
|
||||
num_questions: int = 1319
|
||||
use_chat_completions: bool = False
|
||||
enforce_eager: bool = False
|
||||
disable_flashinfer_sampler: bool = False
|
||||
|
||||
|
||||
QWEN3_DFLASH = DFlashCorrectnessConfig(
|
||||
model="Qwen/Qwen3-8B",
|
||||
draft_model="z-lab/Qwen3-8B-DFlash-b16",
|
||||
expected_accuracy=0.8,
|
||||
expected_acceptance_len=3.5,
|
||||
)
|
||||
|
||||
LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig(
|
||||
model="poolside/Laguna-XS-2.1-NVFP4",
|
||||
draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
|
||||
expected_accuracy=0.7, # Standard GSM8K sanity floor.
|
||||
expected_acceptance_len=3.55 * 0.9,
|
||||
num_speculative_tokens=15,
|
||||
max_model_len=8192,
|
||||
max_num_seqs=32,
|
||||
num_questions=200,
|
||||
use_chat_completions=True,
|
||||
enforce_eager=True,
|
||||
disable_flashinfer_sampler=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mrv2", [False, True])
|
||||
def test_dflash_reference_acceptance_lengths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
use_mrv2: bool,
|
||||
):
|
||||
run_acceptance_length_eval(
|
||||
monkeypatch,
|
||||
spec_config={
|
||||
"model": QWEN3_DFLASH.model,
|
||||
"trust_remote_code": True,
|
||||
"speculative_config": {
|
||||
"method": "dflash",
|
||||
"model": QWEN3_DFLASH.draft_model,
|
||||
"num_speculative_tokens": QWEN3_DFLASH.num_speculative_tokens,
|
||||
"max_model_len": 32768,
|
||||
},
|
||||
"max_model_len": 32768,
|
||||
"max_num_seqs": 128,
|
||||
"gpu_memory_utilization": 0.85,
|
||||
"enforce_eager": False,
|
||||
"disable_log_stats": False,
|
||||
},
|
||||
# Table 1 in https://arxiv.org/pdf/2602.06036.
|
||||
expected_acceptance_lengths={
|
||||
"mt-bench": 4.24,
|
||||
"humaneval": 6.50,
|
||||
"gsm8k": 6.54 * 0.975,
|
||||
},
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
use_mrv2=use_mrv2,
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
@pytest.mark.parametrize(
|
||||
("config", "use_mrv2"),
|
||||
[
|
||||
pytest.param(
|
||||
QWEN3_DFLASH,
|
||||
False,
|
||||
id="qwen3-mrv1",
|
||||
),
|
||||
pytest.param(
|
||||
QWEN3_DFLASH,
|
||||
True,
|
||||
id="qwen3-mrv2",
|
||||
),
|
||||
pytest.param(
|
||||
LAGUNA_DFLASH_NVFP4,
|
||||
True,
|
||||
id="laguna-nvfp4-mrv2",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_dflash_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
config: DFlashCorrectnessConfig,
|
||||
use_mrv2: bool,
|
||||
):
|
||||
"""Guard GSM8K accuracy and batched acceptance length for DFlash models."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
if config.disable_flashinfer_sampler:
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
|
||||
spec_llm = LLM(
|
||||
model=config.model,
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dflash",
|
||||
"model": config.draft_model,
|
||||
"num_speculative_tokens": config.num_speculative_tokens,
|
||||
"max_model_len": config.max_model_len,
|
||||
},
|
||||
max_model_len=config.max_model_len,
|
||||
max_num_seqs=config.max_num_seqs,
|
||||
gpu_memory_utilization=0.85,
|
||||
enforce_eager=config.enforce_eager,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
results = evaluate_gsm8k_offline(
|
||||
spec_llm,
|
||||
num_questions=config.num_questions,
|
||||
use_chat_completions=config.use_chat_completions,
|
||||
)
|
||||
accuracy = results["accuracy"]
|
||||
acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
|
||||
print(
|
||||
f"{config.model}: GSM8K accuracy={accuracy:.3f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}"
|
||||
)
|
||||
|
||||
assert accuracy >= config.expected_accuracy
|
||||
assert acceptance_len >= config.expected_acceptance_len
|
||||
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ...utils import compute_acceptance_len, compute_acceptance_rate
|
||||
|
||||
|
||||
def test_gemma4_dspark_correctness_and_acceptance_rate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
E2E test for Gemma4 DSpark speculative decoding: acceptance rate/length
|
||||
regression coverage plus GSM8K correctness, at temperature=1.0 to exercise
|
||||
the probabilistic draft-sampling/rejection-sampling path (not just greedy).
|
||||
|
||||
gemma-4-12B is instruct-tuned, so GSM8K is run through the chat template
|
||||
(use_chat_completions=True; raw few-shot completion collapses to a few
|
||||
percent). Reference: measured over 5 runs of 200 GSM8K questions at
|
||||
temperature=1.0 (prefix caching disabled):
|
||||
accuracy: min=0.900 max=0.955 mean=0.937
|
||||
acceptance_rate: min=0.578 max=0.595 mean=0.588
|
||||
acceptance_len: min=5.044 max=5.167 mean=5.116
|
||||
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
|
||||
spec_llm = LLM(
|
||||
model="RedHatAI/gemma-4-12B-it-NVFP4",
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dspark",
|
||||
"model": "deepseek-ai/dspark_gemma4_12b_block7",
|
||||
"num_speculative_tokens": 7,
|
||||
"draft_sample_method": "probabilistic",
|
||||
},
|
||||
max_model_len=4096,
|
||||
max_num_seqs=32,
|
||||
gpu_memory_utilization=0.85,
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
try:
|
||||
results = evaluate_gsm8k_offline(
|
||||
spec_llm, num_questions=200, temperature=1.0, use_chat_completions=True
|
||||
)
|
||||
gsm8k_accuracy = results["accuracy"]
|
||||
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
acceptance_len = compute_acceptance_len(metrics)
|
||||
print(
|
||||
f"Gemma4 DSpark acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
|
||||
)
|
||||
|
||||
assert acceptance_rate >= 0.58 * 0.9
|
||||
assert acceptance_len >= 5.0 * 0.9
|
||||
assert gsm8k_accuracy >= 0.92 * 0.9
|
||||
finally:
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_config():
|
||||
target_model = "Qwen/Qwen3-4B-FP8"
|
||||
draft_model = "deepseek-ai/dspark_qwen3_4b_block7"
|
||||
|
||||
return dict(
|
||||
model=target_model,
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "dspark",
|
||||
"model": draft_model,
|
||||
"num_speculative_tokens": 7,
|
||||
"attention_backend": "FLASH_ATTN",
|
||||
"draft_sample_method": "probabilistic",
|
||||
},
|
||||
max_model_len=4096,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
|
||||
def test_dspark_correctness_and_acceptance_rate(dspark_config):
|
||||
"""
|
||||
E2E test for DSpark speculative decoding: acceptance rate/length
|
||||
regression coverage plus GSM8K correctness, at temperature=1.0 to
|
||||
exercise the probabilistic draft-sampling/rejection-sampling path
|
||||
(not just greedy).
|
||||
|
||||
Uses Qwen/Qwen3-4B-FP8 as target with the dspark_qwen3_4b_block7 draft
|
||||
model. Reference: measured over 12 runs of the full GSM8K set at
|
||||
temperature=1.0 (prefix caching disabled to avoid cross-run reuse):
|
||||
accuracy: min=0.782 max=0.814 mean=0.801
|
||||
acceptance_rate: min=0.418 max=0.434 mean=0.428
|
||||
acceptance_len: min=3.928 max=4.037 mean=3.994
|
||||
Thresholds set conservatively to 10% to avoid flaking due to unlucky sampling
|
||||
"""
|
||||
spec_llm = LLM(**dspark_config)
|
||||
|
||||
results = evaluate_gsm8k_offline(spec_llm, temperature=1.0)
|
||||
gsm8k_accuracy = results["accuracy"]
|
||||
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
acceptance_len = compute_acceptance_len(metrics)
|
||||
|
||||
print(
|
||||
f"DSpark acceptance_rate={acceptance_rate:.2f}, "
|
||||
f"acceptance_len={acceptance_len:.2f}, "
|
||||
f"gsm8k_accuracy={gsm8k_accuracy:.3f}"
|
||||
)
|
||||
|
||||
assert acceptance_rate >= 0.428 * 0.9
|
||||
assert acceptance_len >= 3.994 * 0.9
|
||||
assert gsm8k_accuracy >= 0.801 * 0.9
|
||||
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ...utils import compute_acceptance_rate
|
||||
|
||||
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
spec_llm = LLM(
|
||||
model=target_model,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
spec_llm.generate(prompts, sampling_config)
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from ..utils import run_acceptance_length_eval
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mrv2", [False, True])
|
||||
def test_gemma4_mtp_acceptance_lengths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
use_mrv2: bool,
|
||||
):
|
||||
run_acceptance_length_eval(
|
||||
monkeypatch,
|
||||
spec_config={
|
||||
"model": "google/gemma-4-E4B-it",
|
||||
"trust_remote_code": True,
|
||||
"speculative_config": {
|
||||
"method": "mtp",
|
||||
"model": "google/gemma-4-E4B-it-assistant",
|
||||
"num_speculative_tokens": 2,
|
||||
"max_model_len": 32768,
|
||||
},
|
||||
"max_model_len": 32768,
|
||||
"limit_mm_per_prompt": {"image": 0, "audio": 0},
|
||||
"disable_log_stats": False,
|
||||
},
|
||||
expected_acceptance_lengths={
|
||||
"mt-bench": 2.28,
|
||||
"humaneval": 2.68,
|
||||
"gsm8k": 2.67 * 0.975,
|
||||
},
|
||||
chat_template_kwargs={},
|
||||
use_mrv2=use_mrv2,
|
||||
)
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ...utils import compute_acceptance_len, get_test_prompts
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_synthetic_acceptance_rate():
|
||||
"""Verify that synthetic rejection sampling produces an acceptance
|
||||
length close to the requested mean acceptance length."""
|
||||
num_spec_tokens = 3
|
||||
expected_acceptance_len = 1.875
|
||||
tolerance = 0.15
|
||||
|
||||
spec_llm = LLM(
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
trust_remote_code=True,
|
||||
speculative_config={
|
||||
"method": "eagle3",
|
||||
"model": "nm-testing/Llama3_2_1B_speculator.eagle3",
|
||||
"num_speculative_tokens": num_spec_tokens,
|
||||
"max_model_len": 2048,
|
||||
"rejection_sample_method": "synthetic",
|
||||
"synthetic_acceptance_length": expected_acceptance_len,
|
||||
},
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
|
||||
test_prompts = get_test_prompts(mm_enabled=False, num_prompts=50)
|
||||
spec_llm.chat(
|
||||
test_prompts,
|
||||
SamplingParams(temperature=0, max_tokens=64, ignore_eos=True),
|
||||
)
|
||||
|
||||
metrics = spec_llm.get_metrics()
|
||||
acceptance_len = compute_acceptance_len(metrics)
|
||||
|
||||
print(
|
||||
f"Synthetic acceptance length: {acceptance_len:.3f}"
|
||||
f" (expected={expected_acceptance_len:.3f},"
|
||||
f" tolerance=±{tolerance})"
|
||||
)
|
||||
assert abs(acceptance_len - expected_acceptance_len) <= tolerance, (
|
||||
f"Synthetic acceptance length {acceptance_len:.3f} is not within"
|
||||
f" ±{tolerance} of expected {expected_acceptance_len:.3f}"
|
||||
)
|
||||
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from ..utils import compute_acceptance_len
|
||||
|
||||
|
||||
def load_and_process_dataset(data_name: str):
|
||||
from datasets import load_dataset
|
||||
|
||||
if data_name == "gsm8k":
|
||||
dataset = load_dataset("openai/gsm8k", "main", split="test")
|
||||
prompt_fmt = (
|
||||
"{question}\nPlease reason step by step,"
|
||||
" and put your final answer within \\boxed{{}}."
|
||||
)
|
||||
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
|
||||
elif data_name == "mt-bench":
|
||||
dataset = load_dataset("HuggingFaceH4/mt_bench_prompts", split="train")
|
||||
dataset = dataset.map(lambda x: {"turns": x["prompt"]})
|
||||
elif data_name == "humaneval":
|
||||
dataset = load_dataset("openai/openai_humaneval", split="test")
|
||||
prompt_fmt = (
|
||||
"Write a solution to the following problem and make sure"
|
||||
" that it passes the tests:\n```python\n{prompt}\n```"
|
||||
)
|
||||
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
|
||||
|
||||
return dataset
|
||||
|
||||
|
||||
def run_acceptance_length_eval(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
spec_config: dict[str, Any],
|
||||
expected_acceptance_lengths: dict[str, float],
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
use_mrv2: bool,
|
||||
):
|
||||
"""
|
||||
E2E acceptance-rate validation for speculative decoding.
|
||||
|
||||
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
|
||||
through the spec decode engine and asserts the mean acceptance length
|
||||
stays within tolerance of the reference figure for each dataset.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
spec_llm = LLM(**spec_config)
|
||||
|
||||
max_prompts_per_dataset = 200 # mt-bench has 80, humaneval has 164, truncates gsm8k
|
||||
|
||||
tokenizer = spec_llm.get_tokenizer()
|
||||
for dataset_name, expected_len in expected_acceptance_lengths.items():
|
||||
dataset = load_and_process_dataset(dataset_name)
|
||||
prev_metrics = None
|
||||
acceptance_lengths = []
|
||||
for i in tqdm(
|
||||
range(min(max_prompts_per_dataset, len(dataset))),
|
||||
desc=f"Processing {dataset_name}",
|
||||
):
|
||||
user_content = dataset[i]["turns"][0]
|
||||
prompt_text = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": user_content}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# Greedy (temp=0) so acceptance length is deterministic and comparable
|
||||
# across runs.
|
||||
spec_llm.generate(
|
||||
[prompt_text],
|
||||
SamplingParams(temperature=0, max_tokens=2048),
|
||||
use_tqdm=False,
|
||||
)
|
||||
current_metrics = spec_llm.get_metrics()
|
||||
acceptance_len = compute_acceptance_len(current_metrics, prev_metrics)
|
||||
prev_metrics = current_metrics
|
||||
acceptance_lengths.append(acceptance_len)
|
||||
|
||||
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
|
||||
# Fairly tight tolerance of 95% against the reference figures,
|
||||
# watching for regressions. Can be relaxed if test is flaky but be sure to
|
||||
# check for genuine issues such as #40727.
|
||||
expected_len = expected_len * 0.95
|
||||
print(
|
||||
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
|
||||
f" (expected at least {expected_len:.2f})"
|
||||
)
|
||||
|
||||
assert mean_acceptance_length >= expected_len, (
|
||||
f"acceptance_len for {dataset_name} is below expected threshold: "
|
||||
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
|
||||
)
|
||||
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
Reference in New Issue
Block a user