[Feature] [Spec decode]: Enable MLPSpeculator/Medusa and prompt_logprobs with ChunkedPrefill (#10132)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: wallashss <wallashss@ibm.com> Co-authored-by: wallashss <wallashss@ibm.com>
This commit is contained in:
co-authored by
wallashss
parent
2bc3fbba0c
commit
6116ca8cd7
@@ -83,13 +83,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
|
||||
if not non_spec_indices:
|
||||
# All sequence groups in batch have spec decoding enabled
|
||||
contracted = self._contract_batch_all_spec(
|
||||
return self._contract_batch_all_spec(
|
||||
target_sampler_output=target_sampler_output,
|
||||
proposals=proposals,
|
||||
)
|
||||
else:
|
||||
# Batch has a mix of spec decode enabled and disabled seq groups
|
||||
contracted = self._contract_batch(
|
||||
return self._contract_batch(
|
||||
execute_model_req.seq_group_metadata_list,
|
||||
target_sampler_output=target_sampler_output,
|
||||
proposals=proposals,
|
||||
@@ -99,14 +99,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
k=execute_model_req.num_lookahead_slots,
|
||||
)
|
||||
|
||||
all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted
|
||||
return SpeculativeScores(
|
||||
probs=all_probs,
|
||||
token_ids=all_tokens,
|
||||
logprobs=spec_logprobs,
|
||||
hidden_states=all_hidden_states,
|
||||
)
|
||||
|
||||
def _expand_batch(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
@@ -143,13 +135,57 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
|
||||
num_scoring_tokens)
|
||||
|
||||
def _contract_non_speculative(
|
||||
self, scores: SpeculativeScores,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
non_spec_indices: List[int], non_spec_outputs: SpeculativeScores,
|
||||
has_prompt_log: bool) -> SpeculativeScores:
|
||||
"""
|
||||
Augment input `scores` with non-speculative requests outputs.
|
||||
This includes decode requests with speculation turned off, as well
|
||||
as prefill requests when `enable_chunked_prefill` is set.
|
||||
For the latter, prefills are further separated into terminal and
|
||||
non-terminal chunks (from which no token is sampled).
|
||||
"""
|
||||
if not non_spec_indices:
|
||||
return scores
|
||||
|
||||
if has_prompt_log:
|
||||
# When prompt_logprobs is enabled, prefills yield output token
|
||||
# (and respective prob) in the last entry (prompt|out):
|
||||
# [.|.|.|prefill0_out|.|prefill1_out|decode0_out|..].
|
||||
# With chunked prefill, non-terminal chunks have -1 on each
|
||||
# position: they're still picked, but they're discarded later.
|
||||
seq_meta = seq_group_metadata_list
|
||||
nospec_sizes = torch.tensor([
|
||||
seq_meta[i].token_chunk_size if seq_meta[i].is_prompt else 1
|
||||
for i in non_spec_indices
|
||||
])
|
||||
nospec_sampled_token_idxs = torch.cumsum(nospec_sizes, 0).add_(-1)
|
||||
else:
|
||||
# In this case only sampled tokens are returned, select all.
|
||||
nospec_sampled_token_idxs = list(
|
||||
range(len(non_spec_outputs.token_ids)))
|
||||
|
||||
scores.token_ids[non_spec_indices, :1] = \
|
||||
non_spec_outputs.token_ids[nospec_sampled_token_idxs].unsqueeze(1)
|
||||
scores.probs[non_spec_indices, :1, :] = \
|
||||
non_spec_outputs.probs[nospec_sampled_token_idxs].unsqueeze(1)
|
||||
scores.logprobs[non_spec_indices, :1, :] = \
|
||||
non_spec_outputs.logprobs[nospec_sampled_token_idxs].unsqueeze(1)
|
||||
if scores.hidden_states is not None:
|
||||
assert non_spec_outputs.hidden_states is not None
|
||||
scores.hidden_states[non_spec_indices, :1, :] = \
|
||||
non_spec_outputs.hidden_states[nospec_sampled_token_idxs].unsqueeze(1)
|
||||
return scores
|
||||
|
||||
def _contract_batch(
|
||||
self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
|
||||
num_scoring_tokens: int, non_spec_indices: List[int],
|
||||
spec_indices: List[int], k: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
|
||||
Optional[torch.Tensor]]:
|
||||
self,
|
||||
contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
target_sampler_output: SamplerOutput,
|
||||
proposals: SpeculativeProposals, num_scoring_tokens: int,
|
||||
non_spec_indices: List[int], spec_indices: List[int],
|
||||
k: int) -> SpeculativeScores:
|
||||
"""Contract the expanded batch back into its original size.
|
||||
This maps the scores of speculative tokens back to their original
|
||||
sequences.
|
||||
@@ -195,23 +231,28 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
else:
|
||||
all_hidden_states = None
|
||||
|
||||
# Rule out prefills that produce no tokens.
|
||||
non_spec_indices = [
|
||||
idx for idx in non_spec_indices
|
||||
if contracted_seq_group_metadata_list[idx].do_sample
|
||||
]
|
||||
if len(non_spec_indices):
|
||||
all_tokens[non_spec_indices, :1] = \
|
||||
non_spec_target_token_ids.unsqueeze(1)
|
||||
all_probs[non_spec_indices, :1, :] = \
|
||||
non_spec_target_probs.unsqueeze(1)
|
||||
all_logprobs[non_spec_indices, :1, :] = \
|
||||
non_spec_target_logprobs.unsqueeze(1)
|
||||
if all_hidden_states is not None:
|
||||
assert non_spec_target_hidden_states is not None
|
||||
all_hidden_states[non_spec_indices, :1, :] = \
|
||||
non_spec_target_hidden_states.unsqueeze(1)
|
||||
has_prompt_log = any((sg.sampling_params.prompt_logprobs
|
||||
and sg.sampling_params.prompt_logprobs > 0)
|
||||
for sg in contracted_seq_group_metadata_list)
|
||||
# When prompt logprobs is enabled, lens of returned tensors go from
|
||||
# n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
|
||||
# We adjust stride accordingly to get the generated tokens and
|
||||
# their probs, but pass on prompt_logprobs as is.
|
||||
prompt_logprobs = None
|
||||
if (not self._scorer_worker.model_runner.disable_logprobs\
|
||||
and has_prompt_log):
|
||||
prompt_logprobs = [
|
||||
o.prompt_logprobs for o in target_sampler_output.outputs
|
||||
]
|
||||
elif not has_prompt_log:
|
||||
# When prompt logprobs are not to be returned,
|
||||
# we can ignore non-terminal chunks (no out token).
|
||||
non_spec_indices = [
|
||||
idx for idx in non_spec_indices
|
||||
if contracted_seq_group_metadata_list[idx].do_sample
|
||||
]
|
||||
|
||||
# "Contract" speculative.
|
||||
if spec_indices:
|
||||
all_tokens[spec_indices] = target_token_ids
|
||||
all_probs[spec_indices] = target_probs
|
||||
@@ -219,14 +260,27 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
if all_hidden_states is not None:
|
||||
all_hidden_states[spec_indices] = target_hidden_states
|
||||
|
||||
return all_tokens, all_probs, all_logprobs, all_hidden_states
|
||||
spec_scores = SpeculativeScores(probs=all_probs,
|
||||
token_ids=all_tokens,
|
||||
logprobs=all_logprobs,
|
||||
hidden_states=all_hidden_states,
|
||||
prompt_logprobs=prompt_logprobs)
|
||||
|
||||
non_spec_outputs = SpeculativeScores(
|
||||
probs=non_spec_target_probs,
|
||||
token_ids=non_spec_target_token_ids,
|
||||
logprobs=non_spec_target_logprobs,
|
||||
hidden_states=non_spec_target_hidden_states)
|
||||
# Contract remaining nonspec entries based on non_spec_indices, if any.
|
||||
return self._contract_non_speculative(
|
||||
spec_scores, contracted_seq_group_metadata_list, non_spec_indices,
|
||||
non_spec_outputs, has_prompt_log)
|
||||
|
||||
def _contract_batch_all_spec(
|
||||
self,
|
||||
target_sampler_output: SamplerOutput,
|
||||
proposals: SpeculativeProposals,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
|
||||
Optional[torch.Tensor]]:
|
||||
) -> SpeculativeScores:
|
||||
"""Contract the expanded batch back into its original size.
|
||||
This maps the scores of speculative tokens back to their original
|
||||
sequences.
|
||||
@@ -250,8 +304,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
|
||||
target_hidden_states = target_hidden_states.reshape(
|
||||
*target_token_ids.shape, target_hidden_states.shape[-1])
|
||||
|
||||
return (target_token_ids, target_probs, target_logprobs,
|
||||
target_hidden_states)
|
||||
return SpeculativeScores(probs=target_probs,
|
||||
token_ids=target_token_ids,
|
||||
logprobs=target_logprobs,
|
||||
hidden_states=target_hidden_states,
|
||||
prompt_logprobs=None)
|
||||
|
||||
def _create_scoring_model_input(
|
||||
self,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Set, Union
|
||||
from typing import List, Optional, Set, Union
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.sequence import ExecuteModelRequest, PromptLogprobs
|
||||
from vllm.worker.worker_base import WorkerBase
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@ class SpeculativeScores:
|
||||
# Optional last hidden states from the scoring model.
|
||||
hidden_states: Optional[torch.Tensor] = None
|
||||
|
||||
# Scoring model may also return logprobs for prompt tokens
|
||||
# for each request, when chunked prefill is enabled.
|
||||
prompt_logprobs: Optional[List[PromptLogprobs]] = None
|
||||
|
||||
def __repr__(self):
|
||||
return (f"SpeculativeScores("
|
||||
f"probs={self.probs.shape}, "
|
||||
|
||||
@@ -72,9 +72,15 @@ class MQAScorer(SpeculativeScorer):
|
||||
target_token_ids = target_sampler_output.sampled_token_ids
|
||||
target_probs = target_sampler_output.sampled_token_probs
|
||||
target_logprobs = target_sampler_output.logprobs
|
||||
prompt_logprobs = None
|
||||
|
||||
# If all requests have the same number of query tokens, we can avoid
|
||||
# the for loop to build output for better performance.
|
||||
if min(all_proposal_lengths) == k:
|
||||
# Regular decodes only.
|
||||
assert all(not sg.is_prompt
|
||||
for sg in target_seq_group_metadata_list
|
||||
if sg.is_prompt)
|
||||
bs, _ = proposals.proposal_token_ids.shape
|
||||
all_tokens = target_token_ids.reshape(bs, k + 1)
|
||||
all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
|
||||
@@ -88,19 +94,56 @@ class MQAScorer(SpeculativeScorer):
|
||||
all_logprobs = target_logprobs.new_full(size=all_probs.shape,
|
||||
fill_value=-float("inf"))
|
||||
target_token_ids = target_token_ids.flatten()
|
||||
start_loc = 0
|
||||
for i, (proposed_len, seq_meta) in enumerate(
|
||||
zip(all_proposal_lengths, target_seq_group_metadata_list)):
|
||||
|
||||
# When prompt logprobs is enabled, lens of returned tensors go from
|
||||
# n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
|
||||
# We adjust stride accordingly to get the generated tokens and
|
||||
# their probs, but pass on prompt_logprobs as is, since it may be
|
||||
# that n_prompts >> K.
|
||||
has_prompt_log = any((sg.sampling_params.prompt_logprobs
|
||||
and sg.sampling_params.prompt_logprobs > 0)
|
||||
for sg in target_seq_group_metadata_list)
|
||||
# TODO (NickLucche) we should surface `disable_logprobs` as to not
|
||||
# break abstraction to get its value.
|
||||
if (not self._scorer_worker.model_runner.disable_logprobs\
|
||||
and has_prompt_log):
|
||||
prompt_logprobs = [
|
||||
o.prompt_logprobs for o in target_sampler_output.outputs
|
||||
]
|
||||
|
||||
# Split loop into prefill|decode for readability.
|
||||
start_loc, i = 0, 0
|
||||
while i < len(target_seq_group_metadata_list
|
||||
) and target_seq_group_metadata_list[i].is_prompt:
|
||||
seq_meta = target_seq_group_metadata_list[i]
|
||||
end_loc = start_loc
|
||||
if has_prompt_log:
|
||||
end_loc += seq_meta.token_chunk_size
|
||||
elif seq_meta.do_sample:
|
||||
end_loc += 1
|
||||
|
||||
# Skip chunks with no output tokens.
|
||||
if seq_meta.do_sample:
|
||||
output_len = proposed_len + 1
|
||||
end_loc = start_loc + output_len
|
||||
all_tokens[
|
||||
i, :output_len] = target_token_ids[start_loc:end_loc]
|
||||
all_probs[i, :output_len] = target_probs[start_loc:end_loc]
|
||||
all_logprobs[
|
||||
i, :output_len] = target_logprobs[start_loc:end_loc]
|
||||
start_loc = end_loc
|
||||
# Get sampled token (last position in chunk) and its prob.
|
||||
all_tokens[i, 0] = target_token_ids[end_loc - 1]
|
||||
all_probs[i, 0] = target_probs[end_loc - 1]
|
||||
all_logprobs[i, 0] = target_logprobs[end_loc - 1]
|
||||
|
||||
i += 1
|
||||
start_loc = end_loc
|
||||
# Decodes.
|
||||
while i < len(target_seq_group_metadata_list):
|
||||
proposed_len, seq_meta = all_proposal_lengths[
|
||||
i], target_seq_group_metadata_list[i]
|
||||
output_len = proposed_len + 1
|
||||
end_loc = start_loc + output_len
|
||||
all_tokens[
|
||||
i, :output_len] = target_token_ids[start_loc:end_loc]
|
||||
all_probs[i, :output_len] = target_probs[start_loc:end_loc]
|
||||
all_logprobs[
|
||||
i, :output_len] = target_logprobs[start_loc:end_loc]
|
||||
start_loc = end_loc
|
||||
i += 1
|
||||
|
||||
hidden_states = None
|
||||
if target_sampler_output.hidden_states is not None:
|
||||
@@ -110,4 +153,5 @@ class MQAScorer(SpeculativeScorer):
|
||||
return SpeculativeScores(probs=all_probs,
|
||||
token_ids=all_tokens,
|
||||
logprobs=all_logprobs,
|
||||
hidden_states=hidden_states)
|
||||
hidden_states=hidden_states,
|
||||
prompt_logprobs=prompt_logprobs)
|
||||
|
||||
@@ -563,50 +563,57 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
(seq_id, seq_data) for sg in \
|
||||
execute_model_req.seq_group_metadata_list \
|
||||
for seq_id, seq_data in sg.seq_data.items()
|
||||
if sg.do_sample # ignore empty token sequences
|
||||
]
|
||||
completion_seq_group_output_list: List[
|
||||
CompletionSequenceGroupOutput] = []
|
||||
output_index = 0
|
||||
# Make sure the non-terminal prefill chunks are still aligned with
|
||||
# their own empty output.
|
||||
for seq_group_meta in execute_model_req.seq_group_metadata_list:
|
||||
for idx, seq_group_meta in enumerate(
|
||||
execute_model_req.seq_group_metadata_list):
|
||||
needs_prompt_logprobs = seq_output_prompt_logprobs[idx]
|
||||
seq_id, seq_data = seq_data_entries[idx]
|
||||
if needs_prompt_logprobs:
|
||||
prompt_token_ids = seq_data.get_prompt_token_ids()
|
||||
|
||||
# Some of these sequences may belong to non-terminal chunks,
|
||||
# which may still have to report logprobs for prompts.
|
||||
start = 1 if seq_data._num_computed_tokens == 0 \
|
||||
else seq_data._num_computed_tokens
|
||||
end = (seq_data._num_computed_tokens + \
|
||||
seq_group_meta.token_chunk_size)
|
||||
prompt_token_ids = prompt_token_ids[start:end]
|
||||
prompt_logprobs = [
|
||||
create_logprobs_output(
|
||||
token_id=p_token_id,
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
) for p_token_id in prompt_token_ids
|
||||
]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
|
||||
# Since we can get chunks here, we dont always have a sampled token
|
||||
# (only on last chunk) but we still have to provide an output.
|
||||
if not seq_group_meta.do_sample:
|
||||
completion_seq_group_output_list.append(
|
||||
CompletionSequenceGroupOutput(samples=[],
|
||||
prompt_logprobs=None))
|
||||
else:
|
||||
# Sequence with output.
|
||||
seq_id, seq_data = seq_data_entries[output_index]
|
||||
needs_prompt_logprobs = seq_output_prompt_logprobs[
|
||||
output_index]
|
||||
if needs_prompt_logprobs:
|
||||
prompt_token_ids = seq_data.get_prompt_token_ids()
|
||||
prompt_logprobs = [
|
||||
create_logprobs_output(
|
||||
token_id=p_token_id,
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
)
|
||||
# no prompt logprobs for the first token
|
||||
for p_token_id in prompt_token_ids[1:]
|
||||
]
|
||||
else:
|
||||
prompt_logprobs = None
|
||||
completion_seq_group_output_list.append(
|
||||
create_sequence_group_output(
|
||||
token_id=sampled_token_ids_list[output_index][0],
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
seq_id=seq_id,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
prompt_logprobs=prompt_logprobs))
|
||||
output_index += 1
|
||||
CompletionSequenceGroupOutput(
|
||||
samples=[], prompt_logprobs=prompt_logprobs))
|
||||
continue
|
||||
|
||||
# Sequence with output.
|
||||
completion_seq_group_output_list.append(
|
||||
create_sequence_group_output(
|
||||
token_id=sampled_token_ids_list[output_index][0],
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
seq_id=seq_id,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
prompt_logprobs=prompt_logprobs))
|
||||
output_index += 1
|
||||
|
||||
return [SamplerOutput(outputs=completion_seq_group_output_list)]
|
||||
|
||||
@@ -624,24 +631,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
assert len(sampler_output) == 1
|
||||
sampler_output = sampler_output[0]
|
||||
|
||||
# Store hidden states from target model execution.
|
||||
# Store hidden states from target model execution, BxD.
|
||||
hidden_states = sampler_output.hidden_states
|
||||
if hidden_states is not None:
|
||||
# remove hidden_states for prompt tokens
|
||||
# TODO Enable `return_hidden_states`: prefill chunks hidden states
|
||||
# are pruned by the logits processor. Also, they should be arranged
|
||||
# back into full-prefill latent. Address it to enable MLPSpeculator.
|
||||
if any(seq.is_prompt
|
||||
for seq in execute_model_req.seq_group_metadata_list):
|
||||
# Only decodes and prefill terminal chunks need a hidden state.
|
||||
seq_group_meta_with_hidden = [
|
||||
sg for sg in execute_model_req.seq_group_metadata_list
|
||||
if sg.do_sample
|
||||
]
|
||||
if any(seq.is_prompt for seq in seq_group_meta_with_hidden):
|
||||
# Drop hidden_states with no prediction (eg non-terminal chunks)
|
||||
hidden_states = hidden_states[
|
||||
torch.where(sampler_output.sampled_token_ids -
|
||||
VLLM_INVALID_TOKEN_ID)[0]]
|
||||
if self.previous_hidden_states is None:
|
||||
if self.previous_hidden_states is None and len(
|
||||
seq_group_meta_with_hidden):
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
else:
|
||||
self.previous_hidden_states.update(
|
||||
hidden_states, execute_model_req.seq_group_metadata_list)
|
||||
hidden_states, seq_group_meta_with_hidden)
|
||||
elif self.previous_hidden_states and len(
|
||||
seq_group_meta_with_hidden):
|
||||
self.previous_hidden_states.update(hidden_states,
|
||||
seq_group_meta_with_hidden)
|
||||
|
||||
if not skip_proposer:
|
||||
# We prepare the prefill hidden states here so that there no
|
||||
@@ -752,13 +762,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
]
|
||||
if len(non_spec_indices):
|
||||
all_hidden_states = proposal_scores.hidden_states
|
||||
# TODO fix `return_hidden_states`, same as in `_run_no_spec`
|
||||
if all_hidden_states is not None:
|
||||
prefill_hidden_states = all_hidden_states[non_spec_indices]
|
||||
execute_model_req.previous_hidden_states = \
|
||||
prepare_prefill_hidden_states(prefill_hidden_states)
|
||||
# Sync proposer KV cache for prefills.
|
||||
prefill_req = execute_model_req.clone(non_spec_seqs)
|
||||
# TODO avoid sampling here?
|
||||
self.proposer_worker.execute_model(prefill_req)
|
||||
|
||||
with Timer() as verification_timer:
|
||||
@@ -774,6 +784,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
execute_model_req.seq_group_metadata_list,
|
||||
accepted_token_ids,
|
||||
target_logprobs=target_logprobs,
|
||||
prompt_logprobs=proposal_scores.prompt_logprobs
|
||||
if not self._disable_logprobs else None,
|
||||
k=execute_model_req.num_lookahead_slots,
|
||||
stage_times=stage_times)
|
||||
|
||||
@@ -845,19 +857,32 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
# metadata.
|
||||
accepted_token_ids[original_indices] = accepted_token_ids.clone()
|
||||
|
||||
# B x K+1 x D
|
||||
hidden_states = proposal_scores.hidden_states
|
||||
if hidden_states is not None:
|
||||
# Only get terminal hidden states for next step
|
||||
terminal_metadata = [
|
||||
sg for sg in seq_group_metadata_list if sg.do_sample
|
||||
]
|
||||
|
||||
# Contract hidden states based on accepted tokens
|
||||
hs_size = hidden_states.shape[-1]
|
||||
|
||||
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
|
||||
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
|
||||
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
|
||||
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) # b
|
||||
# Drop non-terminal prefill chunks hidden states.
|
||||
hidden_states = hidden_states[
|
||||
accepted_index != VLLM_INVALID_TOKEN_ID]
|
||||
accepted_index = accepted_index[
|
||||
accepted_index != VLLM_INVALID_TOKEN_ID]
|
||||
assert len(accepted_index) == hidden_states.shape[0] == len(
|
||||
terminal_metadata)
|
||||
index = accepted_index[:, None, None].expand(-1, 1,
|
||||
hs_size) # b x 1 x d
|
||||
second_last_token_hidden_states = hidden_states[:, -2] # b x d
|
||||
hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
|
||||
# Store hidden states from target model for subsequent decode step
|
||||
self.previous_hidden_states = HiddenStates(
|
||||
hidden_states, seq_group_metadata_list,
|
||||
hidden_states, terminal_metadata,
|
||||
second_last_token_hidden_states)
|
||||
return accepted_token_ids, logprobs
|
||||
|
||||
@@ -866,6 +891,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
|
||||
target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
|
||||
prompt_logprobs: Optional[
|
||||
torch.Tensor], # shape: [nprompt_tokens, vocab_size]
|
||||
k: int,
|
||||
stage_times: Tuple[float, float, float],
|
||||
) -> List[SamplerOutput]:
|
||||
@@ -909,15 +936,89 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
|
||||
# Construct the output on a per-step, per-sequence basis.
|
||||
# Non-terminal prefill chunks will end up here as rows with just -1s
|
||||
# i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
|
||||
# i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]] while
|
||||
# terminal chunks will only have one generated token at time 0.
|
||||
sampler_output_list: List[SamplerOutput] = []
|
||||
|
||||
# Prefills are not multi-step (return at most 1 token), in order to
|
||||
# avoid padding or repetition to fit decodes, we separate them.
|
||||
for i, sg in enumerate(seq_group_metadata_list):
|
||||
if not sg.is_prompt:
|
||||
# Requests are ordered as prefills|decodes=>no more prefills.
|
||||
break
|
||||
num_logprobs = num_logprobs_per_seq[i]
|
||||
seq_kwargs = dict(token_id=-1,
|
||||
token_id_logprob_rank=0,
|
||||
token_id_logprob=-float('inf'),
|
||||
topk_token_ids=[-1] * num_logprobs,
|
||||
topk_logprobs=[-float('inf')] * num_logprobs,
|
||||
seq_id=seq_ids[i])
|
||||
# Terminal chunk, has token.
|
||||
if sg.do_sample:
|
||||
seq_kwargs.update(
|
||||
dict(
|
||||
token_id=accepted_token_ids[i][0].item(),
|
||||
token_id_logprob_rank=accepted_token_id_ranks_by_step[
|
||||
0][i],
|
||||
token_id_logprob=accepted_token_id_logprobs_by_step[0]
|
||||
[i],
|
||||
topk_token_ids=topk_indices_by_step[0][i]
|
||||
[:num_logprobs],
|
||||
# output only so step is 0
|
||||
topk_logprobs=topk_logprobs_by_step[0][i]
|
||||
[:num_logprobs],
|
||||
))
|
||||
needs_plogs = (sg.sampling_params.prompt_logprobs
|
||||
and sg.sampling_params.prompt_logprobs > 0)
|
||||
plogs = None
|
||||
if prompt_logprobs is not None:
|
||||
# Even non-terminal prompt chunks can have logprobs here.
|
||||
plogs = prompt_logprobs[i]
|
||||
elif needs_plogs:
|
||||
# Prompt logprobs are requested but `_disable_logprobs` is set.
|
||||
seq_data = next(iter(sg.seq_data.values()))
|
||||
# Get only the tokens in this chunk!
|
||||
prompt_token_ids = seq_data.get_prompt_token_ids()
|
||||
prompt_token_ids = prompt_token_ids[
|
||||
seq_data.
|
||||
_num_computed_tokens:seq_data._num_computed_tokens +
|
||||
sg.token_chunk_size]
|
||||
|
||||
is_first_chunk = seq_data._num_computed_tokens == 0
|
||||
# There's no prob generated for the first token in a sequence.
|
||||
if is_first_chunk:
|
||||
prompt_token_ids = prompt_token_ids[1:]
|
||||
plogs = [
|
||||
create_logprobs_output(
|
||||
token_id=p_token_id,
|
||||
token_id_logprob_rank=-1,
|
||||
token_id_logprob=0.0,
|
||||
topk_token_ids=[],
|
||||
topk_logprobs=[],
|
||||
) for p_token_id in prompt_token_ids
|
||||
]
|
||||
seq_kwargs.update(dict(prompt_logprobs=plogs))
|
||||
|
||||
sampler_output_list.append(
|
||||
SamplerOutput(
|
||||
outputs=[create_sequence_group_output(
|
||||
**seq_kwargs)])) # type: ignore
|
||||
|
||||
# Decodes, create one SamplerOutput per-step (at most K+1).
|
||||
for step_index in range(num_steps):
|
||||
if all(token_id == -1
|
||||
for token_id in accepted_token_ids_by_step[step_index]):
|
||||
if all(token_id == -1 for sg, token_id in zip(
|
||||
seq_group_metadata_list,
|
||||
accepted_token_ids_by_step[step_index])
|
||||
if not sg.is_prompt):
|
||||
break
|
||||
|
||||
step_output_token_ids: List[CompletionSequenceGroupOutput] = []
|
||||
for sequence_index in range(batch_size):
|
||||
seq_meta = seq_group_metadata_list[sequence_index]
|
||||
# Prompts already processed above.
|
||||
if seq_meta.is_prompt:
|
||||
continue
|
||||
|
||||
# Each sequence may have a different num_logprobs; retrieve it.
|
||||
num_logprobs = num_logprobs_per_seq[sequence_index]
|
||||
step_output_token_ids.append(
|
||||
@@ -952,6 +1053,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
# This is periodic because the rejection sampler emits metrics
|
||||
# periodically.
|
||||
self._maybe_log_stage_times(*stage_times)
|
||||
# First `n_prefills` entries will contain prefills SamplerOutput when
|
||||
# chunked prefill is enabled, the rest is decodes in multi-step format.
|
||||
return sampler_output_list
|
||||
|
||||
def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,
|
||||
|
||||
Reference in New Issue
Block a user