Added peagle speculators support (#41826)

Signed-off-by: shanjiaz <zsjwpianpian@gmail.com>
This commit is contained in:
shanjiaz
2026-05-12 07:55:57 -07:00
committed by GitHub
parent 7a9cc5e7f0
commit 6ccb10d794
6 changed files with 139 additions and 50 deletions
+5 -5
View File
@@ -84,19 +84,19 @@ steps:
commands: commands:
- pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
- label: DFlash Speculators Correctness - label: Speculators Correctness
key: dflash-speculators-correctness key: speculators-correctness
timeout_in_minutes: 30 timeout_in_minutes: 60
device: h100 device: h100
optional: true optional: true
num_devices: 1 num_devices: 1
source_file_dependencies: source_file_dependencies:
- vllm/v1/spec_decode/ - vllm/v1/spec_decode/
- vllm/model_executor/models/qwen3_dflash.py - vllm/model_executor/models/qwen3_dflash.py
- tests/v1/spec_decode/test_speculators_dflash.py - tests/v1/spec_decode/test_speculators_correctness.py
commands: commands:
- export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
- pytest -v -s v1/spec_decode/test_speculators_dflash.py -m slow_test - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
- label: Spec Decode MTP hybrid (B200) - label: Spec Decode MTP hybrid (B200)
timeout_in_minutes: 30 timeout_in_minutes: 30
+15
View File
@@ -1493,6 +1493,21 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Qwen/Qwen3-VL-8B-Instruct", "Qwen/Qwen3-VL-8B-Instruct",
speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3", speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3",
), ),
# [PEagle]
"PEagleDraftModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
trust_remote_code=True,
speculative_model="nm-testing/qwen3-8b-peagle-speculators",
tokenizer="Qwen/Qwen3-8B",
use_original_num_layers=True,
),
"PeagleLlamaForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-8B",
trust_remote_code=True,
speculative_model="nm-testing/qwen3-8b-peagle-speculators",
tokenizer="Qwen/Qwen3-8B",
use_original_num_layers=True,
),
# [MTP] # [MTP]
"DeepSeekMTPModel": _HfExamplesInfo( "DeepSeekMTPModel": _HfExamplesInfo(
"luccafong/deepseek_mtp_main_random", "luccafong/deepseek_mtp_main_random",
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import pytest import pytest
import torch import torch
@@ -9,22 +11,55 @@ from vllm import LLM
from vllm.config import SpeculativeConfig from vllm.config import SpeculativeConfig
from vllm.distributed import cleanup_dist_env_and_memory from vllm.distributed import cleanup_dist_env_and_memory
MODEL_PATH = "nm-testing/dflash-qwen3-8b-speculators"
EXPECTED_GSM8K_ACCURACY = 0.885 @dataclasses.dataclass
ACCURACY_RTOL = 0.03 class SpeculatorTestConfig:
EXPECTED_ACCEPTANCE_LEN = 3.45 model_path: str
ACCEPTANCE_LEN_RTOL = 0.15 method: str
display_name: str
# Expected per-position acceptance rates (accepted_at_pos / num_drafts) expected_gsm8k_accuracy: float
# Based on GSM8K evaluation with Qwen3-8B dflash speculators. accuracy_rtol: float
EXPECTED_PER_POS_ACCEPTANCE_RATES = [0.795, 0.611, 0.429, 0.282] expected_acceptance_len: float
PER_POS_RTOL = 0.15 acceptance_len_rtol: float
expected_per_pos_acceptance_rates: tuple[float, ...]
per_pos_rtol: float
quantization: str | None = None
parallel_drafting: bool | None = None
def compute_spec_decode_stats( DFLASH_CONFIG = SpeculatorTestConfig(
metrics, model_path="nm-testing/dflash-qwen3-8b-speculators",
) -> dict: method="dflash",
display_name="DFlash",
expected_gsm8k_accuracy=0.885,
accuracy_rtol=0.03,
expected_acceptance_len=3.45,
acceptance_len_rtol=0.15,
expected_per_pos_acceptance_rates=(0.795, 0.611, 0.429, 0.282),
per_pos_rtol=0.15,
quantization="fp8",
)
PEAGLE_CONFIG = SpeculatorTestConfig(
model_path="nm-testing/qwen3-8b-peagle-speculators",
method="eagle3",
display_name="PEagle",
expected_gsm8k_accuracy=0.88,
accuracy_rtol=0.05,
expected_acceptance_len=2.27,
acceptance_len_rtol=0.20,
expected_per_pos_acceptance_rates=(0.66, 0.36, 0.18, 0.09),
per_pos_rtol=0.20,
parallel_drafting=True,
)
SPECULATOR_CONFIGS = [
pytest.param(DFLASH_CONFIG, id="dflash"),
pytest.param(PEAGLE_CONFIG, id="peagle"),
]
def compute_spec_decode_stats(metrics) -> dict:
"""Extract all spec-decode metrics and compute derived stats.""" """Extract all spec-decode metrics and compute derived stats."""
name2metric = {m.name: m for m in metrics} name2metric = {m.name: m for m in metrics}
@@ -67,25 +102,26 @@ def print_spec_decode_stats(stats: dict) -> None:
print("===============================\n") print("===============================\n")
def test_dflash_speculators_model(vllm_runner, example_prompts, monkeypatch): @pytest.mark.parametrize("config", SPECULATOR_CONFIGS)
def test_speculators_model(vllm_runner, example_prompts, monkeypatch, config):
""" """
Test DFlash speculators model properly initializes speculative decoding. Test speculators model properly initializes speculative decoding.
Verifies: Verifies:
1. Speculative config is automatically initialized from speculators config 1. Speculative config is automatically initialized from speculators config
2. Method is detected as 'dflash' 2. Method is detected correctly
3. The draft model path is correctly set 3. parallel_drafting is set correctly (if applicable)
4. Speculative tokens count is valid (num_speculative_tokens=8) 4. The draft model path is correctly set
5. Text generation works with speculative decoding enabled 5. Speculative tokens count is valid
6. Text generation works with speculative decoding enabled
""" """
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner( runner_kwargs = dict(dtype=torch.bfloat16, enforce_eager=True)
MODEL_PATH, if config.quantization:
dtype=torch.bfloat16, runner_kwargs["quantization"] = config.quantization
enforce_eager=True,
quantization="fp8", with vllm_runner(config.model_path, **runner_kwargs) as vllm_model:
) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config vllm_config = vllm_model.llm.llm_engine.vllm_config
assert isinstance(vllm_config.speculative_config, SpeculativeConfig), ( assert isinstance(vllm_config.speculative_config, SpeculativeConfig), (
@@ -93,40 +129,43 @@ def test_dflash_speculators_model(vllm_runner, example_prompts, monkeypatch):
) )
spec_config = vllm_config.speculative_config spec_config = vllm_config.speculative_config
assert spec_config.method == "dflash", ( assert spec_config.method == config.method, (
f"Expected method='dflash', got '{spec_config.method}'" f"Expected method='{config.method}', got '{spec_config.method}'"
) )
if config.parallel_drafting is not None:
assert spec_config.parallel_drafting is config.parallel_drafting, (
f"Expected parallel_drafting={config.parallel_drafting} "
f"for {config.display_name} model"
)
assert spec_config.num_speculative_tokens > 0, ( assert spec_config.num_speculative_tokens > 0, (
f"Expected positive speculative tokens, " f"Expected positive speculative tokens, "
f"got {spec_config.num_speculative_tokens}" f"got {spec_config.num_speculative_tokens}"
) )
assert spec_config.model == MODEL_PATH, ( assert spec_config.model == config.model_path, (
f"Draft model should be {MODEL_PATH}, got {spec_config.model}" f"Draft model should be {config.model_path}, got {spec_config.model}"
) )
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens=20) vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens=20)
assert vllm_outputs, f"No outputs generated for speculators model {MODEL_PATH}" assert vllm_outputs, (
f"No outputs generated for speculators model {config.model_path}"
)
@pytest.mark.slow_test @pytest.mark.slow_test
@large_gpu_mark(min_gb=40) @large_gpu_mark(min_gb=40)
def test_dflash_speculators_correctness(monkeypatch): @pytest.mark.parametrize("config", SPECULATOR_CONFIGS)
def test_speculators_correctness(monkeypatch, config):
""" """
E2E correctness test for DFlash via the speculators auto-detect path. E2E correctness test via the speculators auto-detect path.
Evaluates GSM8k accuracy to ensure the speculators-format model produces Evaluates GSM8k accuracy to ensure the speculators-format model produces
correct outputs, and checks that acceptance length does not collapse under correct outputs, and checks that acceptance length does not collapse under
batched inference (lm-eval style). batched inference (lm-eval style).
Observed per-position acceptance rates on GSM8K (1319 prompts):
pos 0: 0.795, pos 1: 0.611, pos 2: 0.429, pos 3: 0.282,
pos 4: 0.169, pos 5: 0.093, pos 6: 0.048, pos 7: 0.023
Observed mean AL: 3.45 (GSM8K dataset, max_num_seqs=128)
""" """
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
spec_llm = LLM( spec_llm = LLM(
model=MODEL_PATH, model=config.model_path,
trust_remote_code=True, trust_remote_code=True,
max_model_len=4096, max_model_len=4096,
max_num_seqs=128, max_num_seqs=128,
@@ -137,7 +176,7 @@ def test_dflash_speculators_correctness(monkeypatch):
results = evaluate_gsm8k_offline(spec_llm) results = evaluate_gsm8k_offline(spec_llm)
accuracy = results["accuracy"] accuracy = results["accuracy"]
accuracy_threshold = EXPECTED_GSM8K_ACCURACY * (1 - ACCURACY_RTOL) accuracy_threshold = config.expected_gsm8k_accuracy * (1 - config.accuracy_rtol)
assert accuracy >= accuracy_threshold, ( assert accuracy >= accuracy_threshold, (
f"Expected GSM8K accuracy >= {accuracy_threshold:.3f}, got {accuracy:.3f}" f"Expected GSM8K accuracy >= {accuracy_threshold:.3f}, got {accuracy:.3f}"
) )
@@ -147,19 +186,18 @@ def test_dflash_speculators_correctness(monkeypatch):
print_spec_decode_stats(stats) print_spec_decode_stats(stats)
acceptance_len = stats["acceptance_len"] acceptance_len = stats["acceptance_len"]
al_threshold = EXPECTED_ACCEPTANCE_LEN * (1 - ACCEPTANCE_LEN_RTOL) al_threshold = config.expected_acceptance_len * (1 - config.acceptance_len_rtol)
assert acceptance_len >= al_threshold, ( assert acceptance_len >= al_threshold, (
f"DFlash speculators acceptance length too low: " f"{config.display_name} speculators acceptance length too low: "
f"{acceptance_len:.2f} < {al_threshold:.2f}" f"{acceptance_len:.2f} < {al_threshold:.2f}"
) )
# Check per-position acceptance rates for the first few positions.
per_pos_rates = stats["per_pos_acceptance_rates"] per_pos_rates = stats["per_pos_acceptance_rates"]
for i, expected_rate in enumerate(EXPECTED_PER_POS_ACCEPTANCE_RATES): for i, expected_rate in enumerate(config.expected_per_pos_acceptance_rates):
assert i < len(per_pos_rates), ( assert i < len(per_pos_rates), (
f"Missing per-position acceptance rate for position {i}" f"Missing per-position acceptance rate for position {i}"
) )
threshold = expected_rate * (1 - PER_POS_RTOL) threshold = expected_rate * (1 - config.per_pos_rtol)
assert per_pos_rates[i] >= threshold, ( assert per_pos_rates[i] >= threshold, (
f"Per-position acceptance rate at pos {i} too low: " f"Per-position acceptance rate at pos {i} too low: "
f"{per_pos_rates[i]:.4f} < {threshold:.4f} " f"{per_pos_rates[i]:.4f} < {threshold:.4f} "
+2
View File
@@ -591,6 +591,8 @@ _SPECULATIVE_DECODING_MODELS = {
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"), "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"), "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"), "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"PeagleLlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
@@ -43,6 +43,37 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None:
] ]
@register_speculator("peagle")
def update_peagle(config_dict: dict, pre_trained_config: dict) -> None:
"""
Apply PEagle (Parallel Eagle) specific configuration transformations to
the `dict` used to construct the Transformers PreTrainedConfig.
PEagle specific fields:
- draft_vocab_size: Size of the draft model's vocabulary
- target_hidden_size: Hidden size of the target model
- norm_before_residual: Whether to apply norm before residual connection
- norm_before_fc: Whether to apply RMSNorm before the fc projection
- mask_token_id (required): Token ID used for parallel drafting mask
placeholders, mapped to pard_token for the proposer
- eagle_aux_hidden_state_layer_ids: Layer indices from the target model
whose intermediate hidden states are used as auxiliary inputs
"""
pre_trained_config["architectures"] = ["PeagleLlamaForCausalLM"]
pre_trained_config["draft_vocab_size"] = config_dict.get("draft_vocab_size")
if config_dict.get("target_hidden_size") is not None:
pre_trained_config["target_hidden_size"] = config_dict["target_hidden_size"]
pre_trained_config["norm_before_residual"] = config_dict.get(
"norm_before_residual", False
)
pre_trained_config["norm_before_fc"] = config_dict.get("norm_before_fc", False)
pre_trained_config["pard_token"] = config_dict["mask_token_id"]
if config_dict.get("eagle_aux_hidden_state_layer_ids"):
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = config_dict[
"eagle_aux_hidden_state_layer_ids"
]
@register_speculator("dflash") @register_speculator("dflash")
def update_dflash(config_dict: dict, pre_trained_config: dict) -> None: def update_dflash(config_dict: dict, pre_trained_config: dict) -> None:
""" """
@@ -131,7 +131,10 @@ class SpeculatorsConfig(PretrainedConfig):
) )
# Build base vLLM speculative configuration # Build base vLLM speculative configuration
return { result = {
"method": config_dict.get("speculators_model_type"), "method": config_dict.get("speculators_model_type"),
"num_speculative_tokens": num_speculative_tokens, "num_speculative_tokens": num_speculative_tokens,
} }
if result["method"] == "peagle":
result.update({"method": "eagle3", "parallel_drafting": True})
return result