[Speculative Decoding] EAGLE Implementation with Top-1 proposer (#6830)
This commit is contained in:
@@ -203,6 +203,7 @@ class TP1DraftModelRunner(ModelRunner):
|
||||
self,
|
||||
model_input: ModelInputForGPUWithSamplingMetadata,
|
||||
kv_caches: List[torch.Tensor],
|
||||
previous_hidden_states: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
num_steps: int = 1,
|
||||
) -> Optional[List[SamplerOutput]]:
|
||||
@@ -280,13 +281,30 @@ class TP1DraftModelRunner(ModelRunner):
|
||||
graph_batch_size = model_input.input_tokens.shape[0]
|
||||
model_executable = (self.graph_runners[model_input.virtual_engine]
|
||||
[graph_batch_size])
|
||||
|
||||
if previous_hidden_states is not None:
|
||||
hidden_states = torch.cat([
|
||||
previous_hidden_states,
|
||||
torch.empty([
|
||||
graph_batch_size - previous_hidden_states.shape[0],
|
||||
*previous_hidden_states.shape[1:]
|
||||
],
|
||||
dtype=previous_hidden_states.dtype,
|
||||
device=previous_hidden_states.device)
|
||||
])
|
||||
else:
|
||||
hidden_states = None
|
||||
else:
|
||||
model_executable = self.model
|
||||
hidden_states = previous_hidden_states
|
||||
|
||||
outputs: List[SamplerOutput] = []
|
||||
for step in range(num_steps):
|
||||
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
|
||||
|
||||
kwargs = {"previous_hidden_states": hidden_states} \
|
||||
if previous_hidden_states is not None else {}
|
||||
|
||||
# Run model
|
||||
hidden_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
@@ -296,6 +314,7 @@ class TP1DraftModelRunner(ModelRunner):
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalInputs.as_kwargs(multi_modal_kwargs,
|
||||
device=self.device),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Compute the logits.
|
||||
|
||||
@@ -4,8 +4,8 @@ from typing import Dict, List, Set, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
|
||||
SequenceGroupMetadata)
|
||||
from vllm.sequence import (ExecuteModelRequest, HiddenStates, SamplerOutput,
|
||||
SequenceData, SequenceGroupMetadata)
|
||||
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
|
||||
from vllm.spec_decode.interfaces import (SpeculativeProposals,
|
||||
SpeculativeProposer)
|
||||
@@ -157,6 +157,12 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
|
||||
|
||||
updated_execute_model_req.seq_group_metadata_list =\
|
||||
updated_seq_group_metadata_list
|
||||
|
||||
if isinstance(updated_execute_model_req.previous_hidden_states,
|
||||
HiddenStates):
|
||||
updated_execute_model_req.previous_hidden_states\
|
||||
.expand_with_bonus_tokens(seq_with_bonus_token_in_last_step)
|
||||
|
||||
return updated_execute_model_req, indices_of_original_sequence_groups
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -147,6 +147,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
draft_worker_kwargs[
|
||||
"model_runner_cls"] = TP1DraftModelRunner
|
||||
else:
|
||||
if draft_worker_kwargs[
|
||||
"model_config"].hf_config.model_type == "eagle":
|
||||
raise NotImplementedError(
|
||||
"EAGLE does not support TP > 1 yet")
|
||||
|
||||
allow_zero_draft_token_step = False
|
||||
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
|
||||
|
||||
@@ -355,14 +360,34 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
execute_model_req)
|
||||
num_lookahead_slots = execute_model_req.num_lookahead_slots
|
||||
|
||||
# Speculative decoding is disabled in the following cases:
|
||||
# 1. Prefill phase: Speculative decoding is not
|
||||
# used during the prefill phase.
|
||||
# 2. Auto-disable enabled: The running queue size exceeds
|
||||
# the specified threshold.
|
||||
# 3. No request: There are no requests in the batch.
|
||||
# In any of these cases, the proposer and scorer workers
|
||||
# are called normally.
|
||||
no_spec = num_lookahead_slots == 0 or len(
|
||||
execute_model_req.seq_group_metadata_list
|
||||
) == 0 or disable_all_speculation
|
||||
|
||||
# Broadcast how many lookahead slots are scheduled for this step, and
|
||||
# whether all speculation is disabled, to all non-driver workers.
|
||||
|
||||
# This is required as if the number of draft model runs changes
|
||||
# dynamically, the non-driver workers won't know unless we perform a
|
||||
# communication to inform them.
|
||||
|
||||
# no_spec is used to signal non-driver worker about prefill vs decode
|
||||
# stage. This is needed to ensure that order of execution of proposer
|
||||
# and scorer is same in both driver and non-driver workers (i.e.,
|
||||
# scorer -> proposer for prefill and proposer -> scorer in decode). This
|
||||
# order is needed to support models like EAGLE that take scorer states
|
||||
# as inputs.
|
||||
broadcast_dict = dict(
|
||||
num_lookahead_slots=num_lookahead_slots,
|
||||
no_spec=no_spec,
|
||||
disable_all_speculation=disable_all_speculation,
|
||||
)
|
||||
broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
|
||||
@@ -373,17 +398,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
self._maybe_disable_speculative_tokens(
|
||||
disable_all_speculation, execute_model_req.seq_group_metadata_list)
|
||||
|
||||
# Speculative decoding is disabled in the following cases:
|
||||
# 1. Prefill phase: Speculative decoding is not
|
||||
# used during the prefill phase.
|
||||
# 2. Auto-disable enabled: The running queue size exceeds
|
||||
# the specified threshold.
|
||||
# 3. No request: There are no requests in the batch.
|
||||
# In any of these cases, the proposer and scorer workers
|
||||
# are called normally.
|
||||
if num_lookahead_slots == 0 or len(
|
||||
execute_model_req.seq_group_metadata_list
|
||||
) == 0 or disable_all_speculation:
|
||||
if no_spec:
|
||||
return self._run_no_spec(execute_model_req,
|
||||
skip_proposer=disable_all_speculation)
|
||||
return self._run_speculative_decoding_step(execute_model_req,
|
||||
@@ -464,8 +479,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
not called, meaning that the kv-cache in proposer for requests is not
|
||||
updated, so they cannot enable spec decode in the rest decoding.
|
||||
"""
|
||||
if not skip_proposer:
|
||||
self.proposer_worker.execute_model(execute_model_req)
|
||||
|
||||
sampler_output = self.scorer_worker.execute_model(execute_model_req)
|
||||
assert len(sampler_output) == 1
|
||||
@@ -476,10 +489,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
if hidden_states is not None:
|
||||
if self.previous_hidden_states is None:
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
execute_model_req.seq_group_metadata_list, hidden_states)
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
else:
|
||||
self.previous_hidden_states.update(
|
||||
execute_model_req.seq_group_metadata_list, hidden_states)
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
|
||||
if not skip_proposer:
|
||||
# We prepare the prefill hidden states here so that there no
|
||||
# additional complexity in worker for spec_decode vs non_spec_decode
|
||||
# flow and execute_model doesn't need additional modifications.
|
||||
execute_model_req.previous_hidden_states = \
|
||||
prepare_prefill_hidden_states(
|
||||
sampler_output.prefill_hidden_states)
|
||||
|
||||
self.proposer_worker.execute_model(execute_model_req)
|
||||
|
||||
sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
|
||||
execute_model_req=execute_model_req, sampler_output=sampler_output)
|
||||
@@ -507,15 +530,23 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
return False
|
||||
num_lookahead_slots = data["num_lookahead_slots"]
|
||||
|
||||
# Even if num_lookahead_slots is zero, we want to run the proposer model
|
||||
# as it may have KV.
|
||||
#
|
||||
# We run the proposer once per lookahead slot. In the future we should
|
||||
# delegate how many times it runs to the proposer.
|
||||
for _ in range(max(num_lookahead_slots, 1)):
|
||||
self.proposer_worker.execute_model()
|
||||
# In case of prefill, scorer_worker has to be run before proposer so
|
||||
# that the hidden states can be propagated to proposer when needed.
|
||||
if data["no_spec"]:
|
||||
self.scorer_worker.execute_model()
|
||||
|
||||
if not data["disable_all_speculation"]:
|
||||
# Even if num_lookahead_slots is zero, we want to run the
|
||||
# proposer model as it may have KV.
|
||||
#
|
||||
# We run the proposer once per lookahead slot. In the future we
|
||||
# should delegate how many times it runs to the proposer.
|
||||
for _ in range(max(num_lookahead_slots, 1)):
|
||||
self.proposer_worker.execute_model()
|
||||
|
||||
if not data["no_spec"]:
|
||||
self.scorer_worker.execute_model()
|
||||
|
||||
self.scorer_worker.execute_model()
|
||||
return True
|
||||
|
||||
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
|
||||
@@ -546,6 +577,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
raise RuntimeError("Cannot handle cases where distributed draft "
|
||||
"workers generate no tokens")
|
||||
|
||||
execute_model_req.previous_hidden_states = None
|
||||
|
||||
with Timer() as scoring_timer:
|
||||
proposal_scores = self.scorer.score_proposals(
|
||||
execute_model_req,
|
||||
@@ -651,10 +684,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
|
||||
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
|
||||
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
|
||||
second_last_token_hidden_states = hidden_states[:, -2] # b x d
|
||||
hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
|
||||
# Store hidden states from target model for subsequent decode step
|
||||
self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
|
||||
hidden_states)
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
hidden_states, seq_group_metadata_list,
|
||||
second_last_token_hidden_states)
|
||||
|
||||
return accepted_token_ids, logprobs
|
||||
|
||||
@@ -951,3 +986,15 @@ def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
|
||||
(proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
|
||||
|
||||
return new_num_gpu_blocks
|
||||
|
||||
|
||||
def prepare_prefill_hidden_states(
|
||||
prefill_hidden_states: torch.Tensor) -> HiddenStates:
|
||||
# For prefill step in proposer, we run the model for N-1 tokens
|
||||
# because Nth token will be processed in the first decode step. For
|
||||
# N-1 tokens, the input should be 0:N-1 hidden states which should
|
||||
# be concatanated with 1:N token (since output of scorer has to be
|
||||
# the input for proposer). Therefore, we shift the hidden states to
|
||||
# align n-1th hidden state with nth token.
|
||||
return HiddenStates(prefill_hidden_states.roll(
|
||||
shifts=1, dims=0)) if prefill_hidden_states is not None else None
|
||||
|
||||
Reference in New Issue
Block a user