[Model Runner V2] Support Eagle3 (no CUDA graph) (#35029)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -9,7 +9,7 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
if speculative_config.use_eagle():
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle import EagleSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
|
||||
|
||||
return EagleSpeculator(vllm_config, device)
|
||||
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import cast
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsEagle3, supports_eagle3
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def set_eagle3_aux_hidden_state_layers(
|
||||
model: nn.Module,
|
||||
spec_config: SpeculativeConfig,
|
||||
) -> None:
|
||||
if not supports_eagle3(model):
|
||||
raise RuntimeError("Model does not support EAGLE3 interface")
|
||||
# mypy may infer the class-level overload for supports_eagle3.
|
||||
# Narrow explicitly to the runtime protocol instance.
|
||||
if isinstance(model, type):
|
||||
raise RuntimeError("Expected model instance for EAGLE3 configuration")
|
||||
eagle3_model = cast(SupportsEagle3, model)
|
||||
|
||||
aux_layers = get_eagle3_aux_layers_from_config(spec_config)
|
||||
if aux_layers:
|
||||
logger.info("Using Eagle3 auxiliary layers from config: %s", aux_layers)
|
||||
else:
|
||||
aux_layers = eagle3_model.get_eagle3_aux_hidden_state_layers()
|
||||
logger.info("Using Eagle3 auxiliary layers from model: %s", aux_layers)
|
||||
eagle3_model.set_aux_hidden_state_layers(aux_layers)
|
||||
|
||||
|
||||
def get_eagle3_aux_layers_from_config(
|
||||
spec_config: SpeculativeConfig,
|
||||
) -> tuple[int, ...] | None:
|
||||
if not (spec_config and spec_config.draft_model_config):
|
||||
return None
|
||||
hf_config = spec_config.draft_model_config.hf_config
|
||||
if not hasattr(hf_config, "eagle_aux_hidden_state_layer_ids"):
|
||||
return None
|
||||
layer_ids = hf_config.eagle_aux_hidden_state_layer_ids
|
||||
if layer_ids and isinstance(layer_ids, (list, tuple)):
|
||||
return tuple(layer_ids)
|
||||
return None
|
||||
+3
-14
@@ -9,7 +9,6 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -20,7 +19,8 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle_cudagraph import EagleCudaGraphManager
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import EagleCudaGraphManager
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -73,18 +73,7 @@ class EagleSpeculator:
|
||||
self.cudagraph_manager = EagleCudaGraphManager(vllm_config, device)
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with set_model_tag("eagle_head"):
|
||||
self.model = get_model(
|
||||
vllm_config=self.vllm_config, model_config=self.draft_model_config
|
||||
)
|
||||
|
||||
share_lm_head = True
|
||||
if share_lm_head and hasattr(target_model, "lm_head"):
|
||||
if hasattr(self.model, "lm_head"):
|
||||
del self.model.lm_head
|
||||
self.model.lm_head = target_model.lm_head
|
||||
self.model = load_eagle_model(target_model, self.vllm_config)
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
|
||||
|
||||
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
with set_model_tag("eagle_head"):
|
||||
eagle_model = get_model(
|
||||
vllm_config=vllm_config, model_config=draft_model_config
|
||||
)
|
||||
|
||||
# Share target embeddings when the draft checkpoint does not include
|
||||
# its own vocab embedding table.
|
||||
share_embeddings = True
|
||||
if hasattr(eagle_model, "has_own_embed_tokens"):
|
||||
share_embeddings = not eagle_model.has_own_embed_tokens
|
||||
if share_embeddings:
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
inner_model = getattr(target_language_model, "model", None)
|
||||
target_embed_tokens = None
|
||||
if inner_model is not None:
|
||||
if hasattr(inner_model, "embed_tokens"):
|
||||
target_embed_tokens = inner_model.embed_tokens
|
||||
elif hasattr(inner_model, "embedding"):
|
||||
target_embed_tokens = inner_model.embedding
|
||||
if target_embed_tokens is not None and hasattr(eagle_model, "model"):
|
||||
if hasattr(eagle_model.model, "embed_tokens"):
|
||||
del eagle_model.model.embed_tokens
|
||||
eagle_model.model.embed_tokens = target_embed_tokens
|
||||
|
||||
# Only share target lm_head when the draft model does not own one.
|
||||
share_lm_head = True
|
||||
if hasattr(eagle_model, "has_own_lm_head"):
|
||||
share_lm_head = not eagle_model.has_own_lm_head
|
||||
if share_lm_head and hasattr(target_model, "lm_head"):
|
||||
if hasattr(eagle_model, "lm_head"):
|
||||
del eagle_model.lm_head
|
||||
eagle_model.lm_head = target_model.lm_head
|
||||
|
||||
return eagle_model
|
||||
Reference in New Issue
Block a user