[Spec Decode] Add Qwen3 architecture support for EAGLE3 (#43132)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
@@ -1530,6 +1530,16 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3-VL-8B-Instruct",
|
||||
speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
|
||||
),
|
||||
"Eagle3Qwen3ForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
trust_remote_code=True,
|
||||
speculative_model=(
|
||||
"inference-optimization/"
|
||||
"Qwen3-8B-from-Qwen3-8B_regen-speculators.eagle3-qwen3arch-ckpt1"
|
||||
),
|
||||
tokenizer="Qwen/Qwen3-8B",
|
||||
use_original_num_layers=True,
|
||||
),
|
||||
# [PEagle]
|
||||
"PEagleDraftModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
@@ -1545,6 +1555,15 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
tokenizer="Qwen/Qwen3-8B",
|
||||
use_original_num_layers=True,
|
||||
),
|
||||
"PeagleQwen3ForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
trust_remote_code=True,
|
||||
speculative_model=(
|
||||
"inference-optimization/Qwen3-8B-speculators.peagle-qwen3arch-ckpt4"
|
||||
),
|
||||
tokenizer="Qwen/Qwen3-8B",
|
||||
use_original_num_layers=True,
|
||||
),
|
||||
# [MTP]
|
||||
"DeepSeekMTPModel": _HfExamplesInfo(
|
||||
"luccafong/deepseek_mtp_main_random",
|
||||
|
||||
@@ -53,9 +53,39 @@ PEAGLE_CONFIG = SpeculatorTestConfig(
|
||||
parallel_drafting=True,
|
||||
)
|
||||
|
||||
QWEN3_EAGLE3_CONFIG = SpeculatorTestConfig(
|
||||
model_path=(
|
||||
"inference-optimization/"
|
||||
"Qwen3-8B-from-Qwen3-8B_regen-speculators.eagle3-qwen3arch-ckpt1"
|
||||
),
|
||||
method="eagle3",
|
||||
display_name="Qwen3 Eagle3",
|
||||
expected_gsm8k_accuracy=0.88,
|
||||
accuracy_rtol=0.05,
|
||||
expected_acceptance_len=2.67,
|
||||
acceptance_len_rtol=0.10,
|
||||
expected_per_pos_acceptance_rates=(0.76, 0.55, 0.36),
|
||||
per_pos_rtol=0.10,
|
||||
)
|
||||
|
||||
QWEN3_PEAGLE_CONFIG = SpeculatorTestConfig(
|
||||
model_path="inference-optimization/Qwen3-8B-speculators.peagle-qwen3arch-ckpt4",
|
||||
method="eagle3",
|
||||
display_name="Qwen3 PEagle",
|
||||
expected_gsm8k_accuracy=0.88,
|
||||
accuracy_rtol=0.05,
|
||||
expected_acceptance_len=3.42,
|
||||
acceptance_len_rtol=0.15,
|
||||
expected_per_pos_acceptance_rates=(0.78, 0.59, 0.43, 0.29, 0.18, 0.10, 0.05),
|
||||
per_pos_rtol=0.10,
|
||||
parallel_drafting=True,
|
||||
)
|
||||
|
||||
SPECULATOR_CONFIGS = [
|
||||
pytest.param(DFLASH_CONFIG, id="dflash"),
|
||||
pytest.param(PEAGLE_CONFIG, id="peagle"),
|
||||
pytest.param(QWEN3_EAGLE3_CONFIG, id="qwen3arch_eagle3"),
|
||||
pytest.param(QWEN3_PEAGLE_CONFIG, id="qwen3arch_peagle"),
|
||||
]
|
||||
|
||||
|
||||
@@ -176,6 +206,7 @@ def test_speculators_correctness(monkeypatch, config):
|
||||
|
||||
results = evaluate_gsm8k_offline(spec_llm)
|
||||
accuracy = results["accuracy"]
|
||||
print(f"GSM8K Accuracy: {accuracy:.4f}")
|
||||
accuracy_threshold = config.expected_gsm8k_accuracy * (1 - config.accuracy_rtol)
|
||||
assert accuracy >= accuracy_threshold, (
|
||||
f"Expected GSM8K accuracy >= {accuracy_threshold:.3f}, got {accuracy:.3f}"
|
||||
|
||||
Reference in New Issue
Block a user