[Speculative Decoding] EAGLE Implementation with Top-1 proposer (#6830)
This commit is contained in:
@@ -147,6 +147,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
draft_worker_kwargs[
|
||||
"model_runner_cls"] = TP1DraftModelRunner
|
||||
else:
|
||||
if draft_worker_kwargs[
|
||||
"model_config"].hf_config.model_type == "eagle":
|
||||
raise NotImplementedError(
|
||||
"EAGLE does not support TP > 1 yet")
|
||||
|
||||
allow_zero_draft_token_step = False
|
||||
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
|
||||
|
||||
@@ -355,14 +360,34 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
execute_model_req)
|
||||
num_lookahead_slots = execute_model_req.num_lookahead_slots
|
||||
|
||||
# Speculative decoding is disabled in the following cases:
|
||||
# 1. Prefill phase: Speculative decoding is not
|
||||
# used during the prefill phase.
|
||||
# 2. Auto-disable enabled: The running queue size exceeds
|
||||
# the specified threshold.
|
||||
# 3. No request: There are no requests in the batch.
|
||||
# In any of these cases, the proposer and scorer workers
|
||||
# are called normally.
|
||||
no_spec = num_lookahead_slots == 0 or len(
|
||||
execute_model_req.seq_group_metadata_list
|
||||
) == 0 or disable_all_speculation
|
||||
|
||||
# Broadcast how many lookahead slots are scheduled for this step, and
|
||||
# whether all speculation is disabled, to all non-driver workers.
|
||||
|
||||
# This is required as if the number of draft model runs changes
|
||||
# dynamically, the non-driver workers won't know unless we perform a
|
||||
# communication to inform them.
|
||||
|
||||
# no_spec is used to signal non-driver worker about prefill vs decode
|
||||
# stage. This is needed to ensure that order of execution of proposer
|
||||
# and scorer is same in both driver and non-driver workers (i.e.,
|
||||
# scorer -> proposer for prefill and proposer -> scorer in decode). This
|
||||
# order is needed to support models like EAGLE that take scorer states
|
||||
# as inputs.
|
||||
broadcast_dict = dict(
|
||||
num_lookahead_slots=num_lookahead_slots,
|
||||
no_spec=no_spec,
|
||||
disable_all_speculation=disable_all_speculation,
|
||||
)
|
||||
broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
|
||||
@@ -373,17 +398,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
self._maybe_disable_speculative_tokens(
|
||||
disable_all_speculation, execute_model_req.seq_group_metadata_list)
|
||||
|
||||
# Speculative decoding is disabled in the following cases:
|
||||
# 1. Prefill phase: Speculative decoding is not
|
||||
# used during the prefill phase.
|
||||
# 2. Auto-disable enabled: The running queue size exceeds
|
||||
# the specified threshold.
|
||||
# 3. No request: There are no requests in the batch.
|
||||
# In any of these cases, the proposer and scorer workers
|
||||
# are called normally.
|
||||
if num_lookahead_slots == 0 or len(
|
||||
execute_model_req.seq_group_metadata_list
|
||||
) == 0 or disable_all_speculation:
|
||||
if no_spec:
|
||||
return self._run_no_spec(execute_model_req,
|
||||
skip_proposer=disable_all_speculation)
|
||||
return self._run_speculative_decoding_step(execute_model_req,
|
||||
@@ -464,8 +479,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
not called, meaning that the kv-cache in proposer for requests is not
|
||||
updated, so they cannot enable spec decode in the rest decoding.
|
||||
"""
|
||||
if not skip_proposer:
|
||||
self.proposer_worker.execute_model(execute_model_req)
|
||||
|
||||
sampler_output = self.scorer_worker.execute_model(execute_model_req)
|
||||
assert len(sampler_output) == 1
|
||||
@@ -476,10 +489,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
if hidden_states is not None:
|
||||
if self.previous_hidden_states is None:
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
execute_model_req.seq_group_metadata_list, hidden_states)
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
else:
|
||||
self.previous_hidden_states.update(
|
||||
execute_model_req.seq_group_metadata_list, hidden_states)
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
|
||||
if not skip_proposer:
|
||||
# We prepare the prefill hidden states here so that there no
|
||||
# additional complexity in worker for spec_decode vs non_spec_decode
|
||||
# flow and execute_model doesn't need additional modifications.
|
||||
execute_model_req.previous_hidden_states = \
|
||||
prepare_prefill_hidden_states(
|
||||
sampler_output.prefill_hidden_states)
|
||||
|
||||
self.proposer_worker.execute_model(execute_model_req)
|
||||
|
||||
sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
|
||||
execute_model_req=execute_model_req, sampler_output=sampler_output)
|
||||
@@ -507,15 +530,23 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
return False
|
||||
num_lookahead_slots = data["num_lookahead_slots"]
|
||||
|
||||
# Even if num_lookahead_slots is zero, we want to run the proposer model
|
||||
# as it may have KV.
|
||||
#
|
||||
# We run the proposer once per lookahead slot. In the future we should
|
||||
# delegate how many times it runs to the proposer.
|
||||
for _ in range(max(num_lookahead_slots, 1)):
|
||||
self.proposer_worker.execute_model()
|
||||
# In case of prefill, scorer_worker has to be run before proposer so
|
||||
# that the hidden states can be propagated to proposer when needed.
|
||||
if data["no_spec"]:
|
||||
self.scorer_worker.execute_model()
|
||||
|
||||
if not data["disable_all_speculation"]:
|
||||
# Even if num_lookahead_slots is zero, we want to run the
|
||||
# proposer model as it may have KV.
|
||||
#
|
||||
# We run the proposer once per lookahead slot. In the future we
|
||||
# should delegate how many times it runs to the proposer.
|
||||
for _ in range(max(num_lookahead_slots, 1)):
|
||||
self.proposer_worker.execute_model()
|
||||
|
||||
if not data["no_spec"]:
|
||||
self.scorer_worker.execute_model()
|
||||
|
||||
self.scorer_worker.execute_model()
|
||||
return True
|
||||
|
||||
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
|
||||
@@ -546,6 +577,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
raise RuntimeError("Cannot handle cases where distributed draft "
|
||||
"workers generate no tokens")
|
||||
|
||||
execute_model_req.previous_hidden_states = None
|
||||
|
||||
with Timer() as scoring_timer:
|
||||
proposal_scores = self.scorer.score_proposals(
|
||||
execute_model_req,
|
||||
@@ -651,10 +684,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
|
||||
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
|
||||
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
|
||||
second_last_token_hidden_states = hidden_states[:, -2] # b x d
|
||||
hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
|
||||
# Store hidden states from target model for subsequent decode step
|
||||
self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
|
||||
hidden_states)
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
hidden_states, seq_group_metadata_list,
|
||||
second_last_token_hidden_states)
|
||||
|
||||
return accepted_token_ids, logprobs
|
||||
|
||||
@@ -951,3 +986,15 @@ def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
|
||||
(proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
|
||||
|
||||
return new_num_gpu_blocks
|
||||
|
||||
|
||||
def prepare_prefill_hidden_states(
|
||||
prefill_hidden_states: torch.Tensor) -> HiddenStates:
|
||||
# For prefill step in proposer, we run the model for N-1 tokens
|
||||
# because Nth token will be processed in the first decode step. For
|
||||
# N-1 tokens, the input should be 0:N-1 hidden states which should
|
||||
# be concatanated with 1:N token (since output of scorer has to be
|
||||
# the input for proposer). Therefore, we shift the hidden states to
|
||||
# align n-1th hidden state with nth token.
|
||||
return HiddenStates(prefill_hidden_states.roll(
|
||||
shifts=1, dims=0)) if prefill_hidden_states is not None else None
|
||||
|
||||
Reference in New Issue
Block a user