From 89bd2c14d39075a6109ff188b896b600732dc348 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Sun, 21 Jun 2026 16:55:26 -0400 Subject: [PATCH] [Spec Decode] Add Qwen3 architecture support for EAGLE3 (#43132) Signed-off-by: Benjamin Chislett --- tests/models/registry.py | 19 + .../test_speculators_correctness.py | 31 ++ vllm/model_executor/models/qwen3_eagle3.py | 453 ++++++++++++++++++ vllm/model_executor/models/registry.py | 2 + .../configs/speculators/algos.py | 18 +- vllm/v1/spec_decode/llm_base_proposer.py | 2 + 6 files changed, 523 insertions(+), 2 deletions(-) create mode 100644 vllm/model_executor/models/qwen3_eagle3.py diff --git a/tests/models/registry.py b/tests/models/registry.py index ec2c52db56..e865f8efe8 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1530,6 +1530,16 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "Qwen/Qwen3-VL-8B-Instruct", speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3", ), + "Eagle3Qwen3ForCausalLM": _HfExamplesInfo( + "Qwen/Qwen3-8B", + trust_remote_code=True, + speculative_model=( + "inference-optimization/" + "Qwen3-8B-from-Qwen3-8B_regen-speculators.eagle3-qwen3arch-ckpt1" + ), + tokenizer="Qwen/Qwen3-8B", + use_original_num_layers=True, + ), # [PEagle] "PEagleDraftModel": _HfExamplesInfo( "Qwen/Qwen3-8B", @@ -1545,6 +1555,15 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { tokenizer="Qwen/Qwen3-8B", use_original_num_layers=True, ), + "PeagleQwen3ForCausalLM": _HfExamplesInfo( + "Qwen/Qwen3-8B", + trust_remote_code=True, + speculative_model=( + "inference-optimization/Qwen3-8B-speculators.peagle-qwen3arch-ckpt4" + ), + tokenizer="Qwen/Qwen3-8B", + use_original_num_layers=True, + ), # [MTP] "DeepSeekMTPModel": _HfExamplesInfo( "luccafong/deepseek_mtp_main_random", diff --git a/tests/v1/spec_decode/test_speculators_correctness.py b/tests/v1/spec_decode/test_speculators_correctness.py index e133d9eaf9..5a92fe00f3 100644 --- a/tests/v1/spec_decode/test_speculators_correctness.py +++ b/tests/v1/spec_decode/test_speculators_correctness.py @@ -53,9 +53,39 @@ PEAGLE_CONFIG = SpeculatorTestConfig( parallel_drafting=True, ) +QWEN3_EAGLE3_CONFIG = SpeculatorTestConfig( + model_path=( + "inference-optimization/" + "Qwen3-8B-from-Qwen3-8B_regen-speculators.eagle3-qwen3arch-ckpt1" + ), + method="eagle3", + display_name="Qwen3 Eagle3", + expected_gsm8k_accuracy=0.88, + accuracy_rtol=0.05, + expected_acceptance_len=2.67, + acceptance_len_rtol=0.10, + expected_per_pos_acceptance_rates=(0.76, 0.55, 0.36), + per_pos_rtol=0.10, +) + +QWEN3_PEAGLE_CONFIG = SpeculatorTestConfig( + model_path="inference-optimization/Qwen3-8B-speculators.peagle-qwen3arch-ckpt4", + method="eagle3", + display_name="Qwen3 PEagle", + expected_gsm8k_accuracy=0.88, + accuracy_rtol=0.05, + expected_acceptance_len=3.42, + acceptance_len_rtol=0.15, + expected_per_pos_acceptance_rates=(0.78, 0.59, 0.43, 0.29, 0.18, 0.10, 0.05), + per_pos_rtol=0.10, + parallel_drafting=True, +) + SPECULATOR_CONFIGS = [ pytest.param(DFLASH_CONFIG, id="dflash"), pytest.param(PEAGLE_CONFIG, id="peagle"), + pytest.param(QWEN3_EAGLE3_CONFIG, id="qwen3arch_eagle3"), + pytest.param(QWEN3_PEAGLE_CONFIG, id="qwen3arch_peagle"), ] @@ -176,6 +206,7 @@ def test_speculators_correctness(monkeypatch, config): results = evaluate_gsm8k_offline(spec_llm) accuracy = results["accuracy"] + print(f"GSM8K Accuracy: {accuracy:.4f}") accuracy_threshold = config.expected_gsm8k_accuracy * (1 - config.accuracy_rtol) assert accuracy >= accuracy_threshold, ( f"Expected GSM8K accuracy >= {accuracy_threshold:.3f}, got {accuracy:.3f}" diff --git a/vllm/model_executor/models/qwen3_eagle3.py b/vllm/model_executor/models/qwen3_eagle3.py new file mode 100644 index 0000000000..6b03dfcdbd --- /dev/null +++ b/vllm/model_executor/models/qwen3_eagle3.py @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable + +import torch +import torch.nn as nn +from transformers import Qwen3Config + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM +from vllm.multimodal.inputs import NestedTensors + +from .utils import ( + AutoWeightsLoader, + get_draft_quant_config, + maybe_prefix, + process_eagle_weight, +) + +logger = init_logger(__name__) + + +class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer): + def __init__( + self, + vllm_config: VllmConfig, + prefix: str = "", + config: Qwen3Config | None = None, + layer_idx: int = 0, + ) -> None: + config = config or vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = get_draft_quant_config(vllm_config) + + super().__init__( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ) + + # First layer uses 2*hidden_size (embeds + hidden_states concatenated) + # Subsequent layers use hidden_size (only hidden_states, no embeds) + qkv_input_size = 2 * self.hidden_size if layer_idx == 0 else self.hidden_size + + # Parallel drafting checkpoints may have attention bias enabled + qkv_bias = getattr(config, "attention_bias", False) + + # Override qkv_proj with correct input size and bias setting + self.self_attn.qkv_proj = QKVParallelLinear( + qkv_input_size, + self.self_attn.head_dim, + self.self_attn.total_num_heads, + self.self_attn.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "qkv_proj"), + ) + + self.hidden_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.layer_idx = layer_idx + + if getattr(config, "norm_before_residual", False): + self._residual_norm = self._norm_before_residual + else: + self._residual_norm = self._norm_after_residual + + def _norm_before_residual( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + hidden_states = self.hidden_norm(hidden_states) + residual = hidden_states + return hidden_states, residual + + def _norm_after_residual( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + residual = hidden_states + hidden_states = self.hidden_norm(hidden_states) + return hidden_states, residual + + def forward( + self, + positions: torch.Tensor, + embeds: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.layer_idx == 0: + # First layer: concatenate embeds with hidden_states + embeds = self.input_layernorm(embeds) + hidden_states, residual = self._residual_norm(hidden_states=hidden_states) + hidden_states = torch.cat([embeds, hidden_states], dim=-1) + else: + # Subsequent layers: process hidden_states and residuals only + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + # Self Attention + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + + # Fully Connected + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "hidden_states": 0, + "input_embeds": 0, + } +) +class Qwen3Eagle3Model(nn.Module): + def __init__( + self, + *, + vllm_config: VllmConfig, + start_layer_id: int = 0, + prefix: str = "", + ) -> None: + super().__init__() + self.config = vllm_config.speculative_config.draft_model_config.hf_config + self.vocab_size = self.config.vocab_size + + # Get drafter's quantization config + self.quant_config = get_draft_quant_config(vllm_config) + + eagle_config = getattr(self.config, "eagle_config", None) or {} + if "use_aux_hidden_state" in eagle_config: + self.use_aux_hidden_state = eagle_config["use_aux_hidden_state"] + else: + self.use_aux_hidden_state = True + self.norm_before_fc = bool( + eagle_config.get( + "norm_before_fc", getattr(self.config, "norm_before_fc", False) + ) + ) + self.fc_input_size = self.config.hidden_size + + current_vllm_config = get_current_vllm_config() + + self.embed_tokens = VocabParallelEmbedding( + self.config.vocab_size, + self.config.hidden_size, + prefix=maybe_prefix(prefix, "embed_tokens"), + ) + + self.layers = nn.ModuleList( + [ + Qwen3Eagle3DecoderLayer( + current_vllm_config, + prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"), + config=self.config, + layer_idx=layer_idx, + ) + for layer_idx in range(self.config.num_hidden_layers) + ] + ) + if self.use_aux_hidden_state: + num_aux_features = getattr(self.config, "num_aux_layers", None) + if num_aux_features is None: + num_aux_features = getattr(self.config, "num_aux_hidden_states", None) + if num_aux_features is None: + aux_ids = getattr( + self.config, "eagle_aux_hidden_state_layer_ids", None + ) or eagle_config.get("eagle_aux_hidden_state_layer_ids") + num_aux_features = len(aux_ids) if aux_ids else 3 + self.num_aux_layers = num_aux_features + target_hidden_size = getattr( + self.config, "target_hidden_size", self.config.hidden_size + ) + self.fc_input_size = target_hidden_size * num_aux_features + if self.norm_before_fc: + self.input_norm = RMSNorm( + self.fc_input_size, + eps=self.config.rms_norm_eps, + ) + else: + self.input_norm = None + + use_fc_norm = getattr(self.config, "fc_norm", False) + if use_fc_norm: + self.fc_norm = nn.ModuleList( + [ + RMSNorm(target_hidden_size, eps=self.config.rms_norm_eps) + for _ in range(num_aux_features) + ] + ) + else: + self.fc_norm = None + + self.fc = ReplicatedLinear( + input_size=self.fc_input_size, + output_size=self.config.hidden_size, + bias=False, + params_dtype=vllm_config.model_config.dtype, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, "fc"), + return_bias=False, + ) + + self.norm_output = getattr(self.config, "norm_output", False) + self.norm = RMSNorm( + self.config.hidden_size, + eps=self.config.rms_norm_eps, + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_states: torch.Tensor, + input_embeds: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if input_embeds is None: + input_embeds = self.embed_input_ids(input_ids) + assert hidden_states.shape[-1] == input_embeds.shape[-1] + + residual = None + for layer in self.layers: + hidden_states, residual = layer( + positions=positions, + embeds=input_embeds, + hidden_states=hidden_states, + residual=residual, + ) + hidden_states, hidden_prenorm = self.norm(hidden_states, residual) + + # norm_output variant uses the post-norm hidden states. + aux_output = hidden_states if self.norm_output else hidden_prenorm + + return hidden_states, aux_output + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "midlayer." in name: + name = name.replace("midlayer.", "layers.0.") + # Remapping the name FP8 kv-scale or zero point. + if "scale" in name or "zero_point" in name: + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + nn.Module.__init__(self) + self.config = vllm_config.speculative_config.draft_model_config.hf_config + # Ensure draft_vocab_size is set + # default to the base vocab size when absent + if getattr(self.config, "draft_vocab_size", None) is None: + base_vocab_size = getattr(self.config, "vocab_size", None) + self.config.draft_vocab_size = base_vocab_size + target_layer_num = vllm_config.model_config.get_num_layers( + vllm_config.parallel_config + ) + + # Store target layer count in draft config for + # proper layer_types indexing in draft models + self.config.target_layer_count = target_layer_num + self.model = Qwen3Eagle3Model( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + start_layer_id=target_layer_num, + ) + + logit_scale = getattr(self.config, "logit_scale", 1.0) + self.lm_head = ParallelLMHead( + self.config.draft_vocab_size, + self.config.hidden_size, + quant_config=get_draft_quant_config(vllm_config), + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = LogitsProcessor( + self.config.draft_vocab_size, scale=logit_scale + ) + self.draft_id_to_target_id = nn.Parameter( + torch.zeros(self.config.draft_vocab_size, dtype=torch.long), + requires_grad=False, + ) + + self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting + + if self.use_parallel_drafting: + self.register_buffer( + "mask_hidden", + torch.zeros(1, self.model.fc_input_size), + persistent=False, + ) + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: NestedTensors | None = None, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.model(input_ids, positions, hidden_states, inputs_embeds) + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + if self.draft_id_to_target_id is None: + assert logits.shape[1] == self.config.vocab_size, ( + "Expected logits to have shape " + f"(*, {self.config.vocab_size}), but got {logits.shape}" + ) + return logits + + base = torch.arange(self.config.draft_vocab_size, device=logits.device) + targets = base + self.draft_id_to_target_id + logits_new = logits.new_full( + ( + logits.shape[0], + self.config.vocab_size, + ), + float("-inf"), + ) + logits_new[:, targets] = logits + return logits_new + + def combine_hidden_states( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + if not self.model.use_aux_hidden_state: + return hidden_states + # combine multiple auxiliary hidden states returned by eagle3 + + if self.model.norm_before_fc: + hidden_states = self.model.input_norm(hidden_states) + + # `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm` + # applies separate RMSNorms to each chunk of the hidden states. + if self.model.fc_norm is not None: + chunks = hidden_states.chunk(self.model.num_aux_layers, dim=-1) + hidden_states = torch.cat( + [norm(chunk) for norm, chunk in zip(self.model.fc_norm, chunks)], + dim=-1, + ) + + return self.model.fc(hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + model_weights = {} + includes_draft_id_mapping = False + includes_embed_tokens = False + includes_mask_hidden = False + for name, loaded_weight in weights: + if "t2d" in name: + continue + if "d2t" in name: + name = name.replace("d2t", "draft_id_to_target_id") + includes_draft_id_mapping = True + elif "mask_hidden" in name: + # Load mask_hidden directly into buffer + if not self.use_parallel_drafting: + logger.warning( + "mask_hidden found in weights but " + "model is not configured for parallel drafting. " + "Skipping loading mask_hidden." + ) + continue + self.mask_hidden.copy_(loaded_weight.view(1, -1)) + includes_mask_hidden = True + continue + elif "lm_head" not in name: + name = "model." + name + if "embed_tokens" in name: + includes_embed_tokens = True + model_weights[name] = loaded_weight + process_eagle_weight(self, name) + + if not includes_mask_hidden and self.use_parallel_drafting: + raise ValueError( + "mask_hidden not found in weights but " + "model is configured for parallel drafting. " + "Please provide mask_hidden in the weights." + ) + + skip_substrs = ["mask_hidden"] + if not includes_draft_id_mapping: + skip_substrs.append("draft_id_to_target_id") + if not includes_embed_tokens: + skip_substrs.append("embed_tokens") + if not self.model.use_aux_hidden_state: + skip_substrs.append("fc.") + if not self.model.norm_before_fc: + skip_substrs.append("input_norm.") + loader = AutoWeightsLoader( + self, + skip_prefixes=None, + skip_substrs=skip_substrs, + ) + loader.load_weights(model_weights.items()) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index f6286439e6..5c023b3f41 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -617,6 +617,8 @@ _SPECULATIVE_DECODING_MODELS = { "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"), "Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"), + "Eagle3Qwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"), + "PeagleQwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"), "EagleMistralForCausalLM": ("mistral_eagle", "EagleMistralForCausalLM"), "EagleMistralLarge3ForCausalLM": ( "mistral_large_3_eagle", diff --git a/vllm/transformers_utils/configs/speculators/algos.py b/vllm/transformers_utils/configs/speculators/algos.py index 650f09c39f..0dc3ccce08 100644 --- a/vllm/transformers_utils/configs/speculators/algos.py +++ b/vllm/transformers_utils/configs/speculators/algos.py @@ -36,7 +36,14 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None: "norm_before_residual", True ) pre_trained_config["norm_before_fc"] = config_dict.get("norm_before_fc", False) - pre_trained_config["architectures"] = ["Eagle3LlamaForCausalLM"] + eagle3_arch_map = { + "qwen3": "Eagle3Qwen3ForCausalLM", + "llama": "Eagle3LlamaForCausalLM", + } + model_type = pre_trained_config.get("model_type", "llama") + if model_type not in eagle3_arch_map: + raise ValueError(f"Unsupported model_type {model_type} for Eagle3 speculator") + pre_trained_config["architectures"] = [eagle3_arch_map[model_type]] if config_dict.get("eagle_aux_hidden_state_layer_ids"): pre_trained_config["eagle_aux_hidden_state_layer_ids"] = config_dict[ "eagle_aux_hidden_state_layer_ids" @@ -59,7 +66,6 @@ def update_peagle(config_dict: dict, pre_trained_config: dict) -> None: - eagle_aux_hidden_state_layer_ids: Layer indices from the target model whose intermediate hidden states are used as auxiliary inputs """ - pre_trained_config["architectures"] = ["PeagleLlamaForCausalLM"] pre_trained_config["draft_vocab_size"] = config_dict.get("draft_vocab_size") if config_dict.get("target_hidden_size") is not None: pre_trained_config["target_hidden_size"] = config_dict["target_hidden_size"] @@ -67,6 +73,14 @@ def update_peagle(config_dict: dict, pre_trained_config: dict) -> None: "norm_before_residual", False ) pre_trained_config["norm_before_fc"] = config_dict.get("norm_before_fc", False) + peagle_arch_map = { + "qwen3": "PeagleQwen3ForCausalLM", + "llama": "PeagleLlamaForCausalLM", + } + model_type = pre_trained_config.get("model_type", "llama") + if model_type not in peagle_arch_map: + raise ValueError(f"Unsupported model_type {model_type} for PEagle speculator") + pre_trained_config["architectures"] = [peagle_arch_map[model_type]] pre_trained_config["pard_token"] = config_dict["mask_token_id"] if config_dict.get("eagle_aux_hidden_state_layer_ids"): pre_trained_config["eagle_aux_hidden_state_layer_ids"] = config_dict[ diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index bdc10313f4..9f46cbd242 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -24,6 +24,7 @@ from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausal from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM +from vllm.model_executor.models.qwen3_eagle3 import Eagle3Qwen3ForCausalLM from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d @@ -473,6 +474,7 @@ class SpecDecodeBaseProposer: Eagle3LlamaForCausalLM, Eagle3DeepseekV2ForCausalLM, DFlashQwen3ForCausalLM, + Eagle3Qwen3ForCausalLM, ), ) target_hidden_states = self.model.combine_hidden_states(