[Feature] [Spec decode]: Combine chunked prefill with speculative decoding (#9291)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -90,7 +90,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
else:
|
||||
# Batch has a mix of spec decode enabled and disabled seq groups
|
||||
contracted = self._contract_batch(
|
||||
contracted_bs=len(execute_model_req.seq_group_metadata_list),
|
||||
execute_model_req.seq_group_metadata_list,
|
||||
target_sampler_output=target_sampler_output,
|
||||
proposals=proposals,
|
||||
num_scoring_tokens=num_scoring_tokens,
|
||||
@@ -126,7 +126,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
split_batch_by_proposal_len(
|
||||
seq_group_metadata_list, proposal_lens_list)
|
||||
|
||||
target_seq_group_metadata_list = self._create_scoring_model_input(
|
||||
spec_expanded_seqs = self._create_scoring_model_input(
|
||||
seq_group_metadata_list=spec_seqs,
|
||||
proposal_token_ids=proposal_token_ids_list,
|
||||
# NOTE: We determine the seq ids in the expanded batch using the
|
||||
@@ -135,16 +135,19 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
seq_ids=get_all_seq_ids(seq_group_metadata_list)),
|
||||
)
|
||||
|
||||
num_scoring_tokens = len(target_seq_group_metadata_list)
|
||||
target_seq_group_metadata_list.extend(non_spec_seqs)
|
||||
num_scoring_tokens = len(spec_expanded_seqs)
|
||||
# Batch speculative and non-speculative (e.g. chunked prefill) requests
|
||||
# but make sure order is prefill|decode due to backend requirement.
|
||||
target_seq_group_metadata_list = non_spec_seqs + spec_expanded_seqs
|
||||
|
||||
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
|
||||
num_scoring_tokens)
|
||||
|
||||
def _contract_batch(
|
||||
self, contracted_bs: int, target_sampler_output: SamplerOutput,
|
||||
proposals: SpeculativeProposals, num_scoring_tokens: int,
|
||||
non_spec_indices: List[int], spec_indices: List[int], k: int
|
||||
self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
|
||||
num_scoring_tokens: int, non_spec_indices: List[int],
|
||||
spec_indices: List[int], k: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
|
||||
Optional[torch.Tensor]]:
|
||||
"""Contract the expanded batch back into its original size.
|
||||
@@ -154,6 +157,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
contracted_bs is the original batch size, and the batch size that the
|
||||
target_sampler_output will be contracted to.
|
||||
"""
|
||||
contracted_bs = len(contracted_seq_group_metadata_list)
|
||||
(target_token_ids, target_probs, target_logprobs, target_hidden_states,
|
||||
non_spec_target_token_ids, non_spec_target_probs,
|
||||
non_spec_target_logprobs,
|
||||
@@ -166,8 +170,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
|
||||
# The number of tokens in the expanded batch used for speculation is
|
||||
# equal to the total expanded batch size minus the number of samples for
|
||||
# non-speculative sequences.
|
||||
non_spec_expanded_bs = len(non_spec_target_token_ids)
|
||||
# non-speculative sequences, prefill chunks with no out tokens included
|
||||
non_spec_expanded_bs = len(non_spec_indices)
|
||||
spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
|
||||
|
||||
target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
|
||||
@@ -191,7 +195,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
else:
|
||||
all_hidden_states = None
|
||||
|
||||
if non_spec_indices:
|
||||
# Rule out prefills that produce no tokens.
|
||||
non_spec_indices = [
|
||||
idx for idx in non_spec_indices
|
||||
if contracted_seq_group_metadata_list[idx].do_sample
|
||||
]
|
||||
if len(non_spec_indices):
|
||||
all_tokens[non_spec_indices, :1] = \
|
||||
non_spec_target_token_ids.unsqueeze(1)
|
||||
all_probs[non_spec_indices, :1, :] = \
|
||||
@@ -290,9 +299,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
This function creates K+1 target SequenceGroupMetadata to take
|
||||
advantage of the bonus token.
|
||||
"""
|
||||
assert not input_seq_group_metadata.is_prompt, (
|
||||
"Speculating on "
|
||||
"prompts not yet supported")
|
||||
assert len(input_seq_group_metadata.seq_data) == 1, (
|
||||
"Beam search "
|
||||
"not supported in speculative decoding")
|
||||
@@ -390,27 +396,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
# and non spec sequences) and should be removed in the future. It can be
|
||||
# done by supporting per-sequence proposal lens.
|
||||
#
|
||||
# First samples are from speculative scoring, latter samples are non-
|
||||
# speculative samples.
|
||||
split_sizes = (num_scoring_tokens,
|
||||
sampler_output.sampled_token_ids.numel() -
|
||||
num_scoring_tokens)
|
||||
(spec_probs, non_spec_probs
|
||||
) = sampler_output.sampled_token_probs.split(split_sizes)
|
||||
(spec_sampled_tokens, non_spec_sampled_tokens
|
||||
# First samples are non-speculative, latter samples are from speculative
|
||||
# scoring (prefill|decode order).
|
||||
split_sizes = (sampler_output.sampled_token_ids.numel() -
|
||||
num_scoring_tokens, num_scoring_tokens)
|
||||
(non_spec_probs,
|
||||
spec_probs) = sampler_output.sampled_token_probs.split(split_sizes)
|
||||
(non_spec_sampled_tokens, spec_sampled_tokens
|
||||
) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
|
||||
(
|
||||
spec_logprobs,
|
||||
non_spec_logprobs,
|
||||
) = sampler_output.logprobs.split(split_sizes)
|
||||
(non_spec_logprobs,
|
||||
spec_logprobs) = sampler_output.logprobs.split(split_sizes)
|
||||
|
||||
if sampler_output.hidden_states is not None:
|
||||
(
|
||||
spec_hidden_states,
|
||||
non_spec_hidden_states,
|
||||
) = sampler_output.hidden_states.split(split_sizes)
|
||||
(non_spec_hidden_states, spec_hidden_states
|
||||
) = sampler_output.hidden_states.split(split_sizes)
|
||||
else:
|
||||
spec_hidden_states, non_spec_hidden_states = None, None
|
||||
non_spec_hidden_states, spec_hidden_states = None, None
|
||||
|
||||
return (spec_sampled_tokens, spec_probs, spec_logprobs,
|
||||
spec_hidden_states, non_spec_sampled_tokens, non_spec_probs,
|
||||
|
||||
@@ -21,6 +21,11 @@ class MQAScorer(SpeculativeScorer):
|
||||
all_proposal_lengths = proposals.proposal_lens.tolist()
|
||||
for i, seq_group_metadata in enumerate(
|
||||
execute_model_req.seq_group_metadata_list):
|
||||
if all_proposal_lengths[i] == 0:
|
||||
# Keep prompt seqs untouched (keep computed_tokens for chunks).
|
||||
target_seq_group_metadata_list.append(seq_group_metadata)
|
||||
continue
|
||||
|
||||
seq_data_dict = seq_group_metadata.seq_data
|
||||
assert len(seq_data_dict) == 1
|
||||
seq_id = next(iter(seq_data_dict.keys()))
|
||||
@@ -40,8 +45,7 @@ class MQAScorer(SpeculativeScorer):
|
||||
new_seq_data.update_num_computed_tokens(
|
||||
len(prompt_token_ids) + len(output_token_ids) - 1)
|
||||
|
||||
# Ensure that the new sequence has at least one token
|
||||
# because we only use mqa scorer in the decoding stage.
|
||||
# Ensure that the new decode sequence has at least one token.
|
||||
assert len(output_token_ids) >= 1
|
||||
new_seq_data_dict = {target_seq_id: new_seq_data}
|
||||
|
||||
@@ -54,7 +58,6 @@ class MQAScorer(SpeculativeScorer):
|
||||
target_seq_id: seq_group_metadata.block_tables[seq_id],
|
||||
},
|
||||
lora_request=None,
|
||||
token_chunk_size=1,
|
||||
)
|
||||
target_seq_group_metadata_list.append(new_seq_group_metadata)
|
||||
|
||||
@@ -77,6 +80,7 @@ class MQAScorer(SpeculativeScorer):
|
||||
all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
|
||||
all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size)
|
||||
else:
|
||||
# We either have decodes with different lens or prefill+decodes.
|
||||
all_tokens = target_token_ids.new_full(size=(bs, k + 1),
|
||||
fill_value=-1)
|
||||
all_probs = target_probs.new_zeros(*all_tokens.shape,
|
||||
@@ -85,15 +89,18 @@ class MQAScorer(SpeculativeScorer):
|
||||
fill_value=-float("inf"))
|
||||
target_token_ids = target_token_ids.flatten()
|
||||
start_loc = 0
|
||||
for i, proposed_len in enumerate(all_proposal_lengths):
|
||||
output_len = proposed_len + 1
|
||||
end_loc = start_loc + output_len
|
||||
all_tokens[
|
||||
i, :output_len] = target_token_ids[start_loc:end_loc]
|
||||
all_probs[i, :output_len] = target_probs[start_loc:end_loc]
|
||||
all_logprobs[
|
||||
i, :output_len] = target_logprobs[start_loc:end_loc]
|
||||
start_loc = end_loc
|
||||
for i, (proposed_len, seq_meta) in enumerate(
|
||||
zip(all_proposal_lengths, target_seq_group_metadata_list)):
|
||||
# Skip chunks with no output tokens.
|
||||
if seq_meta.do_sample:
|
||||
output_len = proposed_len + 1
|
||||
end_loc = start_loc + output_len
|
||||
all_tokens[
|
||||
i, :output_len] = target_token_ids[start_loc:end_loc]
|
||||
all_probs[i, :output_len] = target_probs[start_loc:end_loc]
|
||||
all_logprobs[
|
||||
i, :output_len] = target_logprobs[start_loc:end_loc]
|
||||
start_loc = end_loc
|
||||
|
||||
hidden_states = None
|
||||
if target_sampler_output.hidden_states is not None:
|
||||
|
||||
@@ -418,7 +418,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
# none of the requests in the batch have spec decoding enabled.
|
||||
# In any of these cases, the proposer and scorer workers
|
||||
# are called normally.
|
||||
no_spec = num_lookahead_slots == 0 or disable_all_speculation or all(
|
||||
# We expect `num_speculative_tokens` to be None for prefills.
|
||||
no_spec = all(
|
||||
sgm.is_prompt for sgm in execute_model_req.seq_group_metadata_list
|
||||
) or num_lookahead_slots == 0 or disable_all_speculation or all(
|
||||
sgm.num_speculative_tokens == 0
|
||||
for sgm in execute_model_req.seq_group_metadata_list)
|
||||
|
||||
@@ -484,7 +487,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
|
||||
def _serialize_sampler_output_no_logprobs(
|
||||
self, execute_model_req: ExecuteModelRequest,
|
||||
sampler_output: SamplerOutput) -> SamplerOutput:
|
||||
sampler_output: SamplerOutput) -> List[SamplerOutput]:
|
||||
"""
|
||||
Creates and returns a `SamplerOutput` with only the token IDs being
|
||||
serialized to CPU and populated in `CompletionSequenceGroupOutput`.
|
||||
@@ -514,41 +517,56 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
if any(seq_output_prompt_logprobs) else \
|
||||
sampler_output.sampled_token_ids).tolist()
|
||||
|
||||
seq_data_entries = (
|
||||
seq_data_entries = [
|
||||
(seq_id, seq_data) for sg in \
|
||||
execute_model_req.seq_group_metadata_list \
|
||||
for seq_id, seq_data in sg.seq_data.items()
|
||||
)
|
||||
if sg.do_sample # ignore empty token sequences
|
||||
]
|
||||
completion_seq_group_output_list: List[
|
||||
CompletionSequenceGroupOutput] = []
|
||||
for index, ((seq_id, seq_data), needs_prompt_logprobs) in \
|
||||
enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)):
|
||||
if needs_prompt_logprobs:
|
||||
prompt_token_ids = seq_data.get_prompt_token_ids()
|
||||
prompt_logprobs = [
|
||||
create_logprobs_output(
|
||||
token_id=p_token_id,
|
||||
output_index = 0
|
||||
# Make sure the non-terminal prefill chunks are still aligned with
|
||||
# their own empty output.
|
||||
for seq_group_meta in execute_model_req.seq_group_metadata_list:
|
||||
# Since we can get chunks here, we dont always have a sampled token
|
||||
# (only on last chunk) but we still have to provide an output.
|
||||
if not seq_group_meta.do_sample:
|
||||
completion_seq_group_output_list.append(
|
||||
CompletionSequenceGroupOutput(samples=[],
|
||||
prompt_logprobs=None))
|
||||
else:
|
||||
# Sequence with output.
|
||||
seq_id, seq_data = seq_data_entries[output_index]
|
||||
needs_prompt_logprobs = seq_output_prompt_logprobs[
|
||||
output_index]
|
||||
if needs_prompt_logprobs:
|
||||
prompt_token_ids = seq_data.get_prompt_token_ids()
|
||||
prompt_logprobs = [
|
||||
create_logprobs_output(
|
||||
token_id=p_token_id,
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
)
|
||||
# no prompt logprobs for the first token
|
||||
for p_token_id in prompt_token_ids[1:]
|
||||
]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
completion_seq_group_output_list.append(
|
||||
create_sequence_group_output(
|
||||
token_id=sampled_token_ids_list[output_index][0],
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
seq_id=seq_id,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
)
|
||||
# no prompt logprobs for the first token
|
||||
for p_token_id in prompt_token_ids[1:]
|
||||
]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
prompt_logprobs=prompt_logprobs))
|
||||
output_index += 1
|
||||
|
||||
completion_seq_group_output_list.append(
|
||||
create_sequence_group_output(
|
||||
token_id=sampled_token_ids_list[index][0],
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
seq_id=seq_id,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
prompt_logprobs=prompt_logprobs))
|
||||
return SamplerOutput(outputs=completion_seq_group_output_list)
|
||||
return [SamplerOutput(outputs=completion_seq_group_output_list)]
|
||||
|
||||
@nvtx_range("spec_decode_worker._run_no_spec")
|
||||
def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
|
||||
@@ -568,6 +586,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
hidden_states = sampler_output.hidden_states
|
||||
if hidden_states is not None:
|
||||
# remove hidden_states for prompt tokens
|
||||
# TODO Enable `return_hidden_states`: prefill chunks hidden states
|
||||
# are pruned by the logits processor. Also, they should be arranged
|
||||
# back into full-prefill latent. Address it to enable MLPSpeculator.
|
||||
if any(seq.is_prompt
|
||||
for seq in execute_model_req.seq_group_metadata_list):
|
||||
hidden_states = hidden_states[
|
||||
@@ -593,14 +614,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
|
||||
execute_model_req=execute_model_req, sampler_output=sampler_output)
|
||||
if self._disable_logprobs else
|
||||
sampler_output)
|
||||
[sampler_output])
|
||||
|
||||
# Clear device tensors from sampler output. This reduces communication
|
||||
# overhead when the engine runs in a different process than the workers.
|
||||
sampler_output.sampled_token_probs = None
|
||||
sampler_output.sampled_token_ids = None
|
||||
sampler_output.logprobs = None
|
||||
return [sampler_output_to_return]
|
||||
return sampler_output_to_return
|
||||
|
||||
def _run_non_driver_rank(self) -> bool:
|
||||
"""Run proposer and verifier model in non-driver workers. This is used
|
||||
@@ -644,9 +665,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
This invokes the proposer worker to get k speculative tokens for each
|
||||
sequence, then scores each speculative token using the scoring worker.
|
||||
|
||||
When `enable_chunked_prefill` is set, scorer will batch decodes and
|
||||
prefills, while proposer will sync its KV-cache by running an extra
|
||||
forward on prefills.
|
||||
|
||||
Returns a list of SamplerOutput, each containing a single token per
|
||||
sequence.
|
||||
"""
|
||||
# With prefill chunking, expect requests to have prompts first
|
||||
# so that backend gets prefill|decode.
|
||||
assert num_lookahead_slots == execute_model_req.num_lookahead_slots
|
||||
|
||||
# Pass last hidden states from target model to proposer
|
||||
@@ -671,6 +698,25 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
proposals,
|
||||
)
|
||||
|
||||
_, (non_spec_seqs, non_spec_indices) = split_batch_by_proposal_len(
|
||||
execute_model_req.seq_group_metadata_list, proposals.proposal_lens)
|
||||
# With prefill chunking enabled, `non_spec_seqs` contains prefills too:
|
||||
# discard decodes that have already been processed by proposer.
|
||||
non_spec_indices = [
|
||||
idx for idx in non_spec_indices
|
||||
if execute_model_req.seq_group_metadata_list[idx].is_prompt
|
||||
]
|
||||
if len(non_spec_indices):
|
||||
all_hidden_states = proposal_scores.hidden_states
|
||||
# TODO fix `return_hidden_states`, same as in `_run_no_spec`
|
||||
if all_hidden_states is not None:
|
||||
prefill_hidden_states = all_hidden_states[non_spec_indices]
|
||||
execute_model_req.previous_hidden_states = \
|
||||
prepare_prefill_hidden_states(prefill_hidden_states)
|
||||
# Sync proposer KV cache for prefills.
|
||||
prefill_req = execute_model_req.clone(non_spec_seqs)
|
||||
self.proposer_worker.execute_model(prefill_req)
|
||||
|
||||
with Timer() as verification_timer:
|
||||
accepted_token_ids, target_logprobs = self._verify_tokens(
|
||||
execute_model_req.seq_group_metadata_list, proposal_scores,
|
||||
@@ -769,7 +815,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
hidden_states, seq_group_metadata_list,
|
||||
second_last_token_hidden_states)
|
||||
|
||||
return accepted_token_ids, logprobs
|
||||
|
||||
def _create_output_sampler_list(
|
||||
@@ -819,6 +864,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
|
||||
|
||||
# Construct the output on a per-step, per-sequence basis.
|
||||
# Non-terminal prefill chunks will end up here as rows with just -1s
|
||||
# i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
|
||||
sampler_output_list: List[SamplerOutput] = []
|
||||
for step_index in range(num_steps):
|
||||
if all(token_id == -1
|
||||
@@ -861,7 +908,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
# This is periodic because the rejection sampler emits metrics
|
||||
# periodically.
|
||||
self._maybe_log_stage_times(*stage_times)
|
||||
|
||||
return sampler_output_list
|
||||
|
||||
def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,
|
||||
|
||||
@@ -109,7 +109,6 @@ class Top1Proposer(SpeculativeProposer):
|
||||
proposal_probs=proposal_probs,
|
||||
proposal_lens=proposal_lens,
|
||||
no_proposals=maybe_sampler_output is None)
|
||||
|
||||
return proposals
|
||||
|
||||
def _split_by_proposal_len(
|
||||
@@ -127,9 +126,10 @@ class Top1Proposer(SpeculativeProposer):
|
||||
nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
|
||||
nonzero_proposal_len_indices: List[int] = []
|
||||
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
|
||||
# The speculative decoding for this request has been disabled
|
||||
# (e.g. due to high traffic).
|
||||
if seq_group_metadata.num_speculative_tokens == 0:
|
||||
# The speculative decoding for this request has either been disabled
|
||||
# (e.g. due to high traffic) or this is a prompt request.
|
||||
if (seq_group_metadata.is_prompt
|
||||
or seq_group_metadata.num_speculative_tokens == 0):
|
||||
proposal_lens.append(0)
|
||||
continue
|
||||
|
||||
|
||||
Reference in New Issue
Block a user