[Feature][Spec Decode] Simplify the use of Eagle Spec Decode (#12304)
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from typing import Dict, List, Set, Tuple
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData,
|
||||
SequenceGroupMetadata)
|
||||
@@ -386,3 +387,14 @@ class MultiStepWorker(ProposerWorkerBase, DelegateWorkerBase):
|
||||
execute_model_req.seq_group_metadata_list):
|
||||
raise NotImplementedError(
|
||||
"MultiStepWorker does not support beam search.")
|
||||
|
||||
def maybe_load_lm_head_weight(
|
||||
self,
|
||||
lm_head_weight: torch.Tensor,
|
||||
) -> None:
|
||||
weight_loader = getattr(
|
||||
self.worker.model_runner.model_runner.model.lm_head.weight,
|
||||
"weight_loader", default_weight_loader)
|
||||
weight_loader(
|
||||
self.worker.model_runner.model_runner.model.lm_head.weight,
|
||||
lm_head_weight)
|
||||
|
||||
Reference in New Issue
Block a user