[Speculative Decoding] Enabling bonus token in speculative decoding for KV cache based models (#5765)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import copy
|
||||
import weakref
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -51,6 +51,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
|
||||
self,
|
||||
execute_model_req: ExecuteModelRequest,
|
||||
sample_len: int,
|
||||
seq_ids_with_bonus_token_in_last_step: Set[int],
|
||||
) -> Tuple[List[SamplerOutput], bool]:
|
||||
"""Run the model forward pass sample_len times. Returns the list of
|
||||
sampler output, one per model forward pass, along with indicator of
|
||||
@@ -60,44 +61,142 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
|
||||
For multi step worker, this indicator shall be True.
|
||||
"""
|
||||
self._raise_if_unsupported(execute_model_req)
|
||||
|
||||
# Shallow copy input data so modifications (such as appending tokens)
|
||||
# do not cause side-effects.
|
||||
copied_seq_group_metadata_list = self._shallow_copy_inputs(
|
||||
execute_model_req.seq_group_metadata_list)
|
||||
copied_execute_model_req = execute_model_req.clone(
|
||||
copied_seq_group_metadata_list)
|
||||
|
||||
# Expand the batch for sequences with a bonus token.
|
||||
# Perform a forward pass on the expanded batch and filter the
|
||||
# response to retain only the original sequences' responses.
|
||||
expanded_request, indices_of_seq_with_bonus_tokens =\
|
||||
self._expand_execute_model_request(
|
||||
execute_model_req, seq_ids_with_bonus_token_in_last_step)
|
||||
# Run model sample_len times.
|
||||
model_outputs: List[SamplerOutput] = []
|
||||
if isinstance(self.model_runner, TP1DraftModelRunner):
|
||||
copied_execute_model_req.num_steps = sample_len
|
||||
expanded_request.num_steps = sample_len
|
||||
model_outputs = self.execute_model(
|
||||
execute_model_req=copied_execute_model_req)
|
||||
execute_model_req=expanded_request)
|
||||
else:
|
||||
# TODO: Remove this branch once DraftModelRunner supports TP>1.
|
||||
for _ in range(sample_len):
|
||||
model_output: List[SamplerOutput] = super().execute_model(
|
||||
execute_model_req=copied_execute_model_req)
|
||||
execute_model_req=expanded_request)
|
||||
assert (len(model_output) == 1
|
||||
), "composing multistep workers not supported"
|
||||
model_output = model_output[0]
|
||||
|
||||
self._append_new_tokens(model_output,
|
||||
copied_seq_group_metadata_list)
|
||||
self._append_new_tokens(
|
||||
model_output, expanded_request.seq_group_metadata_list)
|
||||
model_outputs.append(model_output)
|
||||
|
||||
return model_outputs, True
|
||||
filtered_model_outputs = self._filter_model_output(
|
||||
model_outputs, indices_of_seq_with_bonus_tokens)
|
||||
return filtered_model_outputs, True
|
||||
|
||||
@staticmethod
|
||||
def _expand_execute_model_request(
|
||||
execute_model_req: ExecuteModelRequest,
|
||||
seq_with_bonus_token_in_last_step: set,
|
||||
) -> Tuple[ExecuteModelRequest, List[int]]:
|
||||
"""
|
||||
Expands the execute model request based on sequences with bonus
|
||||
tokens.
|
||||
|
||||
For each sequence with a bonus token, this method creates a new
|
||||
sequence without the bonus token and adds it to the execute model
|
||||
request. The original sequence groups are also retained. The indices
|
||||
of the original sequence groups are returned for further processing.
|
||||
|
||||
Args:
|
||||
execute_model_req (ExecuteModelRequest): The original execute
|
||||
model request.
|
||||
seq_with_bonus_token_in_last_step (set): Set of sequence IDs that
|
||||
contain bonus tokens.
|
||||
|
||||
Returns:
|
||||
Tuple[ExecuteModelRequest, List[int]]: The updated execute model
|
||||
request with expanded sequences and a list of indices corresponding
|
||||
to the original sequence groups.
|
||||
"""
|
||||
updated_seq_group_metadata_list: List[SequenceGroupMetadata] = []
|
||||
updated_execute_model_req = execute_model_req.clone(
|
||||
updated_seq_group_metadata_list)
|
||||
indices_of_original_sequence_groups = []
|
||||
for seq_group in execute_model_req.seq_group_metadata_list:
|
||||
seq_group_has_bonus_tokens = False
|
||||
for seq_id, _ in seq_group.seq_data.items():
|
||||
# Identify sequences with bonus tokens in the sequence group.
|
||||
if seq_id in seq_with_bonus_token_in_last_step:
|
||||
seq_group_has_bonus_tokens = True
|
||||
break
|
||||
if seq_group_has_bonus_tokens:
|
||||
#Create new sequences without the last bonus token. These new
|
||||
# sequence have the same sequence id as the original sequence.
|
||||
# We create a new sequence group and add them there.
|
||||
updated_seq_group_without_bonus_token = \
|
||||
MultiStepWorker._copy_seq_metadata_excluding_last_token(
|
||||
seq_group, seq_with_bonus_token_in_last_step)
|
||||
updated_seq_group_metadata_list.append(
|
||||
updated_seq_group_without_bonus_token)
|
||||
# Add the original sequence group.
|
||||
updated_seq_group_metadata_list.append(
|
||||
MultiStepWorker._shallow_copy_seq_group_metadata(seq_group))
|
||||
# Record the index of the original sequence group.
|
||||
indices_of_original_sequence_groups.append(
|
||||
len(updated_seq_group_metadata_list) - 1)
|
||||
|
||||
updated_execute_model_req.seq_group_metadata_list =\
|
||||
updated_seq_group_metadata_list
|
||||
return updated_execute_model_req, indices_of_original_sequence_groups
|
||||
|
||||
@staticmethod
|
||||
def _filter_model_output(
|
||||
expanded_batch_outputs: List[SamplerOutput],
|
||||
output_indices_to_retain: List[int]) -> List[SamplerOutput]:
|
||||
"""
|
||||
Filters the model output to include only the specified sequence
|
||||
outputs. This method contracts the expanded batch output from the
|
||||
model to retain the outputs of only those sequences indicated by the
|
||||
provided indices.
|
||||
|
||||
Args:
|
||||
expanded_batch_output (List[SamplerOutput]): The expanded output
|
||||
batch from the model.
|
||||
output_indices_to_retain (List[int]): Indices of the model outputs
|
||||
to retain.
|
||||
|
||||
Returns:
|
||||
List[SamplerOutput]: A list containing the filtered model
|
||||
outputs for the specified indices.
|
||||
"""
|
||||
return [
|
||||
SamplerOutput(
|
||||
outputs=[
|
||||
expanded_batch_output.outputs[i]
|
||||
for i in output_indices_to_retain
|
||||
],
|
||||
sampled_token_probs=(
|
||||
expanded_batch_output.
|
||||
sampled_token_probs[output_indices_to_retain]
|
||||
if expanded_batch_output.sampled_token_probs is not None
|
||||
else None),
|
||||
logprobs=(
|
||||
expanded_batch_output.logprobs[output_indices_to_retain]
|
||||
if expanded_batch_output.logprobs is not None else None),
|
||||
sampled_token_ids=(expanded_batch_output.
|
||||
sampled_token_ids[output_indices_to_retain]
|
||||
if expanded_batch_output.sampled_token_ids
|
||||
is not None else None))
|
||||
for expanded_batch_output in expanded_batch_outputs
|
||||
]
|
||||
|
||||
def get_spec_proposals(
|
||||
self,
|
||||
execute_model_req: ExecuteModelRequest,
|
||||
seq_ids_with_bonus_token_in_last_step: set,
|
||||
) -> SpeculativeProposals:
|
||||
"""Produce speculations given an input batch of sequences. The number of
|
||||
speculative tokens per sequence is determined by max_proposal_len.
|
||||
"""
|
||||
|
||||
return self._proposer.get_spec_proposals(execute_model_req)
|
||||
return self._proposer.get_spec_proposals(
|
||||
execute_model_req, seq_ids_with_bonus_token_in_last_step)
|
||||
|
||||
@staticmethod
|
||||
def _append_new_tokens(
|
||||
@@ -123,9 +222,8 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
|
||||
seq.update_num_computed_tokens(1)
|
||||
|
||||
@staticmethod
|
||||
def _shallow_copy_inputs(
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata]
|
||||
) -> List[SequenceGroupMetadata]:
|
||||
def _shallow_copy_seq_group_metadata(
|
||||
seq_group_metadata: SequenceGroupMetadata, ) -> SequenceGroupMetadata:
|
||||
"""Copy input data structures to remove side-effects when input data
|
||||
structures are shared with other modules.
|
||||
|
||||
@@ -133,26 +231,62 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
|
||||
The alternative is deep-copying (or other form of deep copy); this has
|
||||
performance downsides.
|
||||
"""
|
||||
|
||||
# Shallow-copy the list of SequenceGroupMetadata. This allows us to
|
||||
# Shallow-copy the SequenceGroupMetadata. This allows us to
|
||||
# append tokens and change is_prompt without external side-effects.
|
||||
new_seq_group_metadata_list: List[SequenceGroupMetadata] = []
|
||||
# We must shallow-copy seq_group_metadata as is_prompt could change.
|
||||
new_seq_group_metadata = copy.copy(seq_group_metadata)
|
||||
|
||||
for old_seq_group_metadata in seq_group_metadata_list:
|
||||
# We must shallow-copy seq_group_metadata as is_prompt could change.
|
||||
seq_group_metadata = copy.copy(old_seq_group_metadata)
|
||||
new_seq_group_metadata_list.append(seq_group_metadata)
|
||||
# We must shallow-copy seq_data as we will append token ids
|
||||
new_seq_data: Dict[int, SequenceData] = {}
|
||||
for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
|
||||
new_seq_data[seq_id] = copy.copy(old_seq_data)
|
||||
new_seq_data[seq_id].output_token_ids =\
|
||||
old_seq_data.output_token_ids[:]
|
||||
|
||||
# We must shallow-copy seq_data as we will append token ids
|
||||
new_seq_data: Dict[int, SequenceData] = {}
|
||||
for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
|
||||
new_seq_group_metadata.seq_data = new_seq_data
|
||||
return new_seq_group_metadata
|
||||
|
||||
@staticmethod
|
||||
def _copy_seq_metadata_excluding_last_token(
|
||||
seq_group_metadata: SequenceGroupMetadata,
|
||||
seq_ids_to_copy: Set[int],
|
||||
) -> SequenceGroupMetadata:
|
||||
"""
|
||||
Creates a shallow copy of the given SequenceGroupMetadata, retaining
|
||||
only the sequence IDs specified in seq_ids_to_copy. For each of these
|
||||
sequence IDs, all output_token_ids except the last one are copied.
|
||||
Sequence IDs not in seq_ids_to_copy are excluded from the copy.
|
||||
|
||||
Parameters:
|
||||
seq_group_metadata (SequenceGroupMetadata): The original sequence
|
||||
group metadata.
|
||||
seq_ids_to_copy (Set[int]): The set of sequence IDs to include in the
|
||||
copy.
|
||||
|
||||
Returns:
|
||||
SequenceGroupMetadata: A shallow copy of the sequence group metadata
|
||||
with the specified modifications.
|
||||
"""
|
||||
# Shallow-copy the SequenceGroupMetadata.
|
||||
new_seq_group_metadata = copy.copy(seq_group_metadata)
|
||||
# Shallow-copy seq_data and modify the output_token_ids.
|
||||
new_seq_data: Dict[int, SequenceData] = {}
|
||||
for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
|
||||
if (seq_id in seq_ids_to_copy):
|
||||
new_seq_data[seq_id] = copy.copy(old_seq_data)
|
||||
new_seq_data[
|
||||
seq_id].output_token_ids = old_seq_data.output_token_ids[:]
|
||||
|
||||
seq_group_metadata.seq_data = new_seq_data
|
||||
|
||||
return new_seq_group_metadata_list
|
||||
# Copy all the output token ids except the last.
|
||||
# Also reduce num_computed_tokens by 1 since we are not
|
||||
# including the last output token.
|
||||
# NOTE: num_computed_tokens is not directly used by the
|
||||
# speculative decoding workers, as it is only relevant for
|
||||
# chunked prefill, which is disabled for speculative decoding.
|
||||
# However, to maintain consistency in num_computed_tokens,
|
||||
# we update it here.
|
||||
new_seq_data[seq_id].output_token_ids =\
|
||||
old_seq_data.output_token_ids[:-1]
|
||||
new_seq_data[seq_id].update_num_computed_tokens(-1)
|
||||
new_seq_group_metadata.seq_data = new_seq_data
|
||||
return new_seq_group_metadata
|
||||
|
||||
def _assert_enough_kv_space(
|
||||
self, seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
|
||||
Reference in New Issue
Block a user