[Core] Refactoring sampler and support prompt logprob for chunked prefill (#4309)
This commit is contained in:
@@ -7,11 +7,11 @@ import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.ops.sample import sample as sample_triton
|
||||
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
|
||||
SamplingTensors)
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
SamplingTensors,
|
||||
SequenceGroupToSample)
|
||||
from vllm.sampling_params import SamplingType
|
||||
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
|
||||
SamplerOutput, SequenceData, SequenceGroupOutput,
|
||||
SequenceOutput)
|
||||
SamplerOutput, SequenceGroupOutput, SequenceOutput)
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
@@ -48,11 +48,14 @@ class Sampler(nn.Module):
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
"""
|
||||
Args:
|
||||
logits: (num_tokens, vocab_size).
|
||||
sampling_metadata: Metadata for sampling.
|
||||
"""
|
||||
assert logits is not None
|
||||
_, vocab_size = logits.shape
|
||||
|
||||
# Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
|
||||
# have not been generated yet
|
||||
logits = _apply_min_tokens_penalty(logits, sampling_metadata)
|
||||
|
||||
# Prepare sampling tensors with pinned memory to avoid blocking.
|
||||
@@ -83,7 +86,6 @@ class Sampler(nn.Module):
|
||||
# Compute the probabilities.
|
||||
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
|
||||
# Compute the log probabilities.
|
||||
# Use log_softmax to ensure numerical stability.
|
||||
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
|
||||
|
||||
# Sample the next tokens.
|
||||
@@ -149,24 +151,28 @@ def _apply_min_tokens_penalty(
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
"""Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
|
||||
have not been generated yet
|
||||
"""
|
||||
# list of indices in logits that will be set to -inf
|
||||
logits_to_penalize = []
|
||||
start_idx = 0
|
||||
for i, seq_group in enumerate(sampling_metadata.seq_groups):
|
||||
seq_ids, sampling_params = seq_group
|
||||
logits_applied = 0
|
||||
for seq_group in sampling_metadata.seq_groups:
|
||||
seq_ids = seq_group.seq_ids
|
||||
sampling_params = seq_group.sampling_params
|
||||
|
||||
# handle prompt_logprobs by skipping rows in logits added for the prompt
|
||||
# tokens (prompt logprobs are not penalized)
|
||||
if (i < sampling_metadata.num_prompts
|
||||
and sampling_params.prompt_logprobs is not None):
|
||||
assert len(seq_ids) == 1
|
||||
start_idx += sampling_metadata.prompt_lens[i] - 1
|
||||
sample_indices = seq_group.sample_indices
|
||||
logits_applied += len(sample_indices) + len(
|
||||
seq_group.prompt_logprob_indices)
|
||||
if not seq_group.do_sample:
|
||||
continue
|
||||
|
||||
start_idx = sample_indices[0]
|
||||
min_tokens = sampling_params.min_tokens
|
||||
if min_tokens > 0:
|
||||
seqs_to_penalize = []
|
||||
for i, seq_id in enumerate(seq_ids):
|
||||
seq_data = sampling_metadata.seq_data[seq_id]
|
||||
seq_data = seq_group.seq_data[seq_id]
|
||||
if len(seq_data.output_token_ids) < min_tokens:
|
||||
seqs_to_penalize.append(i)
|
||||
|
||||
@@ -180,15 +186,13 @@ def _apply_min_tokens_penalty(
|
||||
logits_to_penalize.extend(
|
||||
itertools.product(seqs_to_penalize, token_ids_to_penalize))
|
||||
|
||||
start_idx += len(seq_ids)
|
||||
|
||||
if logits_to_penalize:
|
||||
# use zip and * to group indices along each dimension
|
||||
# eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
|
||||
logits[tuple(zip(*logits_to_penalize))] = -float("inf")
|
||||
|
||||
# verifies that no rows in logits were missed unexpectedly
|
||||
assert start_idx == logits.shape[0]
|
||||
assert logits_applied == logits.shape[0]
|
||||
return logits
|
||||
|
||||
|
||||
@@ -265,14 +269,30 @@ def _apply_min_p(
|
||||
|
||||
|
||||
def _greedy_sample(
|
||||
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
|
||||
selected_seq_groups: List[SequenceGroupToSample],
|
||||
samples: torch.Tensor,
|
||||
) -> List[Tuple[List[int], List[int]]]:
|
||||
"""Run greedy sampling on a given samples.
|
||||
|
||||
Args:
|
||||
selected_seq_groups: A list of sequence groups batched.
|
||||
samples: (num_selected_samples,) A tensor of samples. The length of
|
||||
samples could be smaller than selected_seq_groups if
|
||||
seq_group.do_sample is False.
|
||||
Returns:
|
||||
Tuple of (next_token_ids, parent_ids). The length of returned list is
|
||||
same as the length of selected_seq_groups. If the corresponding
|
||||
seq_group has do_sample=False, tuple contains ([], [])
|
||||
"""
|
||||
samples = samples.tolist()
|
||||
sample_idx = 0
|
||||
results = []
|
||||
for seq_group in selected_seq_groups:
|
||||
seq_ids, _ = seq_group
|
||||
if not seq_group.do_sample:
|
||||
results.append(([], []))
|
||||
continue
|
||||
|
||||
seq_ids = seq_group.seq_ids
|
||||
num_parent_seqs = len(seq_ids)
|
||||
assert num_parent_seqs == 1, (
|
||||
"Greedy sampling should have only one seq.")
|
||||
@@ -284,16 +304,33 @@ def _greedy_sample(
|
||||
|
||||
|
||||
def _random_sample(
|
||||
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
|
||||
is_prompts: List[bool],
|
||||
selected_seq_groups: List[SequenceGroupToSample],
|
||||
random_samples: torch.Tensor,
|
||||
) -> List[Tuple[List[int], List[int]]]:
|
||||
"""Run random sampling on a given samples.
|
||||
|
||||
Args:
|
||||
selected_seq_groups: A list of sequence groups batched.
|
||||
random_samples: (num_selected_samples,) A tensor of samples. The
|
||||
length of samples could be smaller than selected_seq_groups if
|
||||
seq_group.do_sample is False.
|
||||
Returns:
|
||||
Tuple of (next_token_ids, parent_ids). The length of returned list is
|
||||
same as the length of selected_seq_groups. If the corresponding
|
||||
seq_group has do_sample=False, tuple contains ([], [])
|
||||
"""
|
||||
# Find the maximum best_of value of the prompt phase requests.
|
||||
random_samples = random_samples.cpu()
|
||||
sample_idx = 0
|
||||
results = []
|
||||
for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
|
||||
seq_ids, sampling_params = seq_group
|
||||
for seq_group in selected_seq_groups:
|
||||
if not seq_group.do_sample:
|
||||
results.append(([], []))
|
||||
continue
|
||||
|
||||
seq_ids = seq_group.seq_ids
|
||||
sampling_params = seq_group.sampling_params
|
||||
is_prompt = seq_group.is_prompt
|
||||
num_parent_seqs = len(seq_ids)
|
||||
if is_prompt:
|
||||
# Prompt phase.
|
||||
@@ -311,11 +348,20 @@ def _random_sample(
|
||||
|
||||
|
||||
def _beam_search_sample(
|
||||
selected_seq_groups: List[Tuple[List[int], SamplingParams]],
|
||||
is_prompts: List[bool],
|
||||
seq_data: Dict[int, SequenceData],
|
||||
selected_seq_groups: List[SequenceGroupToSample],
|
||||
logprobs: torch.Tensor,
|
||||
) -> List[Tuple[List[int], List[int]]]:
|
||||
"""Run beam sampling on a given samples.
|
||||
|
||||
Args:
|
||||
selected_seq_groups: A list of sequence groups batched.
|
||||
logprobs: (num_selected_samples, vocab_size,) A tensor of logprob
|
||||
on selected sample indices.
|
||||
Returns:
|
||||
Tuple of (next_token_ids, parent_ids). The length of returned list is
|
||||
same as the length of selected_seq_groups. If the corresponding
|
||||
seq_group has do_sample=False, tuple contains ([], [])
|
||||
"""
|
||||
# We sample 2 * beam_width candidates to make sure that with high
|
||||
# probability we can get `beam_width` candidates in addition to
|
||||
# the finished sequences for the next iteration. See
|
||||
@@ -327,8 +373,13 @@ def _beam_search_sample(
|
||||
# other sampling methods.
|
||||
sample_idx = 0
|
||||
results = []
|
||||
for seq_group, is_prompt in zip(selected_seq_groups, is_prompts):
|
||||
seq_ids, sampling_params = seq_group
|
||||
for seq_group in selected_seq_groups:
|
||||
if not seq_group.do_sample:
|
||||
results.append(([], []))
|
||||
continue
|
||||
|
||||
is_prompt = seq_group.is_prompt
|
||||
seq_ids, sampling_params = seq_group.seq_ids, seq_group.sampling_params
|
||||
num_parent_seqs = len(seq_ids)
|
||||
beam_width = sampling_params.best_of
|
||||
seq_group_logprobs = logprobs[sample_idx:sample_idx + num_parent_seqs]
|
||||
@@ -343,7 +394,8 @@ def _beam_search_sample(
|
||||
else:
|
||||
# Generation phase.
|
||||
cumulative_logprobs = [
|
||||
seq_data[seq_id].cumulative_logprob for seq_id in seq_ids
|
||||
seq_group.seq_data[seq_id].cumulative_logprob
|
||||
for seq_id in seq_ids
|
||||
]
|
||||
cumulative_logprobs = torch.tensor(
|
||||
cumulative_logprobs,
|
||||
@@ -371,8 +423,7 @@ def _beam_search_sample(
|
||||
def _multinomial(
|
||||
probs: torch.Tensor,
|
||||
num_samples: int,
|
||||
seq_groups: Optional[List[Tuple[List[int], SamplingParams]]] = None,
|
||||
generators: Optional[List[torch.Generator]] = None,
|
||||
seq_groups: Optional[List[SequenceGroupToSample]] = None,
|
||||
) -> torch.Tensor:
|
||||
if num_samples > 1:
|
||||
# This is equivalent to torch.repeat_interleaved (which also
|
||||
@@ -388,9 +439,11 @@ def _multinomial(
|
||||
q.exponential_()
|
||||
else:
|
||||
sample_idx = 0
|
||||
for (seq_ids, _), generator in zip(seq_groups, generators):
|
||||
for seq_group in seq_groups:
|
||||
seq_ids = seq_group.seq_ids
|
||||
next_sample_idx = sample_idx + len(seq_ids) * num_samples
|
||||
q[sample_idx:next_sample_idx].exponential_(generator=generator)
|
||||
q[sample_idx:next_sample_idx].exponential_(
|
||||
generator=seq_group.generator)
|
||||
sample_idx = next_sample_idx
|
||||
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
|
||||
|
||||
@@ -405,7 +458,7 @@ def _sample_with_torch(
|
||||
categorized_seq_group_ids = {t: [] for t in SamplingType}
|
||||
categorized_sample_indices = sampling_metadata.categorized_sample_indices
|
||||
for i, seq_group in enumerate(sampling_metadata.seq_groups):
|
||||
_, sampling_params = seq_group
|
||||
sampling_params = seq_group.sampling_params
|
||||
sampling_type = sampling_params.sampling_type
|
||||
categorized_seq_group_ids[sampling_type].append(i)
|
||||
|
||||
@@ -429,13 +482,11 @@ def _sample_with_torch(
|
||||
num_tokens = len(sample_indices)
|
||||
if num_tokens == 0:
|
||||
continue
|
||||
seq_group_ids = categorized_seq_group_ids[sampling_type]
|
||||
seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
|
||||
is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
|
||||
sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
|
||||
is_prompts, sample_indices)
|
||||
long_sample_indices = sample_indices.long()
|
||||
|
||||
seq_group_id = categorized_seq_group_ids[sampling_type]
|
||||
seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
|
||||
sample_metadata[sampling_type] = (seq_group_id, seq_groups)
|
||||
long_sample_indices = sample_indices.long()
|
||||
if sampling_type == SamplingType.GREEDY:
|
||||
greedy_samples = torch.argmax(logprobs[long_sample_indices],
|
||||
dim=-1)
|
||||
@@ -455,14 +506,13 @@ def _sample_with_torch(
|
||||
|
||||
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
|
||||
max_best_of_in_batch = 1
|
||||
for seq_group, is_prompt in zip(seq_groups, is_prompts):
|
||||
if is_prompt:
|
||||
_, sampling_params = seq_group
|
||||
for seq_group in seq_groups:
|
||||
if seq_group.is_prompt:
|
||||
sampling_params = seq_group.sampling_params
|
||||
max_best_of_in_batch = max(max_best_of_in_batch,
|
||||
sampling_params.best_of)
|
||||
seeded_args = {} if sampling_type == SamplingType.RANDOM else {
|
||||
"seq_groups": seq_groups,
|
||||
"generators": sampling_metadata.generators,
|
||||
}
|
||||
|
||||
multinomial_samples[sampling_type] = _multinomial(
|
||||
@@ -481,25 +531,22 @@ def _sample_with_torch(
|
||||
|
||||
# GPU<->CPU sync happens in the loop below.
|
||||
# This also converts the sample output to Python objects.
|
||||
|
||||
for sampling_type in SamplingType:
|
||||
if sampling_type not in sample_metadata:
|
||||
continue
|
||||
seq_group_ids, seq_groups, is_prompts, sample_indices = sample_metadata[
|
||||
sampling_type]
|
||||
(seq_group_id, seq_groups) = sample_metadata[sampling_type]
|
||||
if sampling_type == SamplingType.GREEDY:
|
||||
sample_results = _greedy_sample(seq_groups, greedy_samples)
|
||||
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
|
||||
sample_results = _random_sample(seq_groups, is_prompts,
|
||||
sample_results = _random_sample(seq_groups,
|
||||
multinomial_samples[sampling_type])
|
||||
elif sampling_type == SamplingType.BEAM:
|
||||
sample_results = _beam_search_sample(seq_groups, is_prompts,
|
||||
sampling_metadata.seq_data,
|
||||
sample_results = _beam_search_sample(seq_groups,
|
||||
beam_search_logprobs)
|
||||
sample_results_dict.update(zip(seq_group_ids, sample_results))
|
||||
sample_results_dict.update(zip(seq_group_id, sample_results))
|
||||
|
||||
sample_results = [
|
||||
sample_results_dict[i]
|
||||
sample_results_dict.get(i, ([], []))
|
||||
for i in range(len(sampling_metadata.seq_groups))
|
||||
]
|
||||
return sample_results, sampled_token_ids_tensor
|
||||
@@ -514,7 +561,7 @@ def _sample_with_triton_kernel(
|
||||
categorized_seq_group_ids = {t: [] for t in SamplingType}
|
||||
categorized_sample_indices = sampling_metadata.categorized_sample_indices
|
||||
for i, seq_group in enumerate(sampling_metadata.seq_groups):
|
||||
_, sampling_params = seq_group
|
||||
sampling_params = seq_group.sampling_params
|
||||
sampling_type = sampling_params.sampling_type
|
||||
categorized_seq_group_ids[sampling_type].append(i)
|
||||
|
||||
@@ -530,17 +577,16 @@ def _sample_with_triton_kernel(
|
||||
num_tokens = len(sample_indices)
|
||||
if num_tokens == 0:
|
||||
continue
|
||||
seq_group_ids = categorized_seq_group_ids[sampling_type]
|
||||
seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_ids]
|
||||
is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
|
||||
sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
|
||||
is_prompts, sample_indices,
|
||||
seq_group_id = categorized_seq_group_ids[sampling_type]
|
||||
seq_groups = [sampling_metadata.seq_groups[i] for i in seq_group_id]
|
||||
sample_metadata[sampling_type] = (seq_group_id, seq_groups,
|
||||
sample_indices,
|
||||
sampled_token_indices)
|
||||
if sampling_type in (SamplingType.GREEDY, SamplingType.RANDOM,
|
||||
SamplingType.RANDOM_SEED):
|
||||
for seq_group, is_prompt in zip(seq_groups, is_prompts):
|
||||
if is_prompt:
|
||||
_, sampling_params = seq_group
|
||||
for seq_group in seq_groups:
|
||||
if seq_group.is_prompt:
|
||||
sampling_params = seq_group.sampling_params
|
||||
max_best_of_in_batch = max(max_best_of_in_batch,
|
||||
sampling_params.best_of)
|
||||
elif sampling_type == SamplingType.BEAM:
|
||||
@@ -564,22 +610,21 @@ def _sample_with_triton_kernel(
|
||||
for sampling_type in SamplingType:
|
||||
if sampling_type not in sample_metadata:
|
||||
continue
|
||||
(seq_group_ids, seq_groups, is_prompts, sample_indices,
|
||||
(seq_group_id, seq_groups, sample_indices,
|
||||
sampled_token_indices) = sample_metadata[sampling_type]
|
||||
if sampling_type == SamplingType.GREEDY:
|
||||
sample_results = _greedy_sample(
|
||||
seq_groups, sampled_tokens[sampled_token_indices][:, 0])
|
||||
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
|
||||
sample_results = _random_sample(
|
||||
seq_groups, is_prompts, sampled_tokens[sampled_token_indices])
|
||||
seq_groups, sampled_tokens[sampled_token_indices])
|
||||
elif sampling_type == SamplingType.BEAM:
|
||||
sample_results = _beam_search_sample(seq_groups, is_prompts,
|
||||
sampling_metadata.seq_data,
|
||||
sample_results = _beam_search_sample(seq_groups,
|
||||
beam_search_logprobs)
|
||||
sample_results_dict.update(zip(seq_group_ids, sample_results))
|
||||
sample_results_dict.update(zip(seq_group_id, sample_results))
|
||||
|
||||
sample_results = [
|
||||
sample_results_dict[i]
|
||||
sample_results_dict.get(i, ([], []))
|
||||
for i in range(len(sampling_metadata.seq_groups))
|
||||
]
|
||||
return sample_results
|
||||
@@ -590,6 +635,18 @@ def _sample(
|
||||
sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
|
||||
include_gpu_probs_tensor: bool, modify_greedy_probs: bool
|
||||
) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
|
||||
"""
|
||||
Args:
|
||||
probs: (num_query_tokens_in_batch, num_vocab)
|
||||
logprobs: (num_query_tokens_in_batch, num_vocab)
|
||||
sampling_metadata: The metadata for a batch for sampling.
|
||||
sampling_tensors: Tensors that include sampling related metadata.
|
||||
|
||||
Returns:
|
||||
(next_token_ids, parent_seq_ids) for each seq group in a batch.
|
||||
If sampling is skipped, it returns ([], [])
|
||||
sampled_token_ids_tensor: A tensor of sampled token ids.
|
||||
"""
|
||||
return _sample_with_torch(
|
||||
probs,
|
||||
logprobs,
|
||||
@@ -626,56 +683,97 @@ def _get_logprobs(
|
||||
logprobs: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
sample_results: List[Tuple[List[int], List[int]]],
|
||||
) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
|
||||
int, float]]]]:
|
||||
# Prepare query indices
|
||||
batched_logprobs_query_seq_indices: List[int] = []
|
||||
batched_logprobs_query_token_indices: List[int] = []
|
||||
# at least get one logprob for each token
|
||||
) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
|
||||
"""Return sample lobprobs and prompt logprobs.
|
||||
|
||||
The logic consists of 3 parts.
|
||||
- Select indices to compute logprob from, ranks of token ids, and
|
||||
the top k token ids from logprobs.
|
||||
- Compute prompt logprobs if required.
|
||||
- Compute sample logprobs if required.
|
||||
|
||||
Args:
|
||||
logprobs: (num_query_tokens_across_batch, num_vocab). Each query token's
|
||||
logprob per vocab. Sequence groups' query tokens are batched in a
|
||||
single flattened tensor. For example, assuming there are N
|
||||
seq groups, it is sorted by prefill tokens for seq_group_1 (if
|
||||
prompt logprob is enabled), decode tokens for seq_group_1 (if
|
||||
sampling is required), prefill tokens for seq_group_2, ...
|
||||
sampling_metadata: The sampling metadata.
|
||||
sample_results: (num_seq_groups) The tuple of (next_token_ids,
|
||||
parent_ids) for each sequence group. When beam search is enabled,
|
||||
sample_results can contain different number of seq_ids from
|
||||
sampling_metadata.seq_groups. It is because beam search creates
|
||||
2 * BEAM_WIDTH number of samples (whereas there are only up to
|
||||
BEAM_WIDTH number of seq_ids).
|
||||
|
||||
Returns:
|
||||
A tuple of prompt and sample logprobs per sequence group in a batch.
|
||||
"""
|
||||
# The index of query token to calculate logprobs. It includes both
|
||||
# prompt and sample logprob indices.
|
||||
query_indices: List[int] = []
|
||||
# The next token ids to get the logprob value from.
|
||||
next_token_ids: List[int] = []
|
||||
# The largest requested number of logprobs. We find logprobs as many as the
|
||||
# largest num logprobs in this API.
|
||||
largest_num_logprobs = 1
|
||||
sample_idx = 0
|
||||
for i, (seq_group, sample_result) in enumerate(
|
||||
zip(sampling_metadata.seq_groups, sample_results)):
|
||||
seq_ids, sampling_params = seq_group
|
||||
next_token_ids, parent_ids = sample_result
|
||||
num_parent_seqs = len(seq_ids)
|
||||
if (i < sampling_metadata.num_prompts
|
||||
|
||||
# Select indices to compute logprob from, ranks of token ids, and the top
|
||||
# k token ids from logprobs.
|
||||
for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
|
||||
sample_results):
|
||||
sampling_params = seq_group.sampling_params
|
||||
|
||||
# Update indices and tokens for prompt logprobs.
|
||||
if (seq_group.is_prompt
|
||||
and sampling_params.prompt_logprobs is not None):
|
||||
largest_num_logprobs = max(largest_num_logprobs,
|
||||
sampling_params.prompt_logprobs)
|
||||
prompt_len = sampling_metadata.prompt_lens[i]
|
||||
prompt_tokens = sampling_metadata.seq_data[
|
||||
seq_ids[0]].prompt_token_ids
|
||||
batched_logprobs_query_seq_indices.extend(
|
||||
sample_idx + j for j in range(prompt_len - 1))
|
||||
batched_logprobs_query_token_indices.extend(
|
||||
token_id for token_id in prompt_tokens[1:])
|
||||
sample_idx += prompt_len - 1
|
||||
batched_logprobs_query_seq_indices.extend(
|
||||
[sample_idx + parent_id for parent_id in parent_ids])
|
||||
batched_logprobs_query_token_indices.extend(next_token_ids)
|
||||
if sampling_params.logprobs is not None:
|
||||
largest_num_logprobs = max(largest_num_logprobs,
|
||||
sampling_params.logprobs)
|
||||
sample_idx += num_parent_seqs
|
||||
assert sample_idx == logprobs.size(0)
|
||||
next_prompt_tokens = _get_next_prompt_tokens(seq_group)
|
||||
query_indices.extend(seq_group.prompt_logprob_indices)
|
||||
next_token_ids.extend(next_prompt_tokens)
|
||||
|
||||
batched_logprobs_query_seq_indices_gpu = torch.tensor(
|
||||
batched_logprobs_query_seq_indices, device=logprobs.device)
|
||||
batched_logprobs_query_token_indices_gpu = torch.tensor(
|
||||
batched_logprobs_query_token_indices, device=logprobs.device)
|
||||
# Update indices and next tokenes for sample logprob.
|
||||
if seq_group.do_sample:
|
||||
token_ids, parent_seq_ids = sample_result
|
||||
# NOTE: We cannot directly use sample_indices because
|
||||
# sample_indices only contain parent seq_ids of a previous step.
|
||||
# The current step may have different number of seq_ids, and
|
||||
# we can obtain it from `sample_result[1]`.
|
||||
query_idx = seq_group.sample_indices[0]
|
||||
query_indices.extend(
|
||||
[query_idx + parent_id for parent_id in parent_seq_ids])
|
||||
next_token_ids.extend(token_ids)
|
||||
|
||||
# Batched query for logprobs of selected token
|
||||
batched_logprobs_query_result = logprobs[[
|
||||
batched_logprobs_query_seq_indices_gpu,
|
||||
batched_logprobs_query_token_indices_gpu
|
||||
if sampling_params.logprobs is not None:
|
||||
largest_num_logprobs = max(largest_num_logprobs,
|
||||
sampling_params.logprobs)
|
||||
|
||||
assert len(next_token_ids) == len(query_indices)
|
||||
|
||||
if len(query_indices) == 0:
|
||||
empty_sampled_logprob = []
|
||||
empty_prompt_logprob = None
|
||||
return [empty_prompt_logprob], [empty_sampled_logprob]
|
||||
|
||||
query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
|
||||
next_token_ids_gpu = torch.tensor(next_token_ids, device=logprobs.device)
|
||||
|
||||
# (num_selected_query_tokens, num_logprobs). Note that query_indices can
|
||||
# contain duplicates if beam search is enabled.
|
||||
selected_logprobs = logprobs[[
|
||||
query_indices_gpu,
|
||||
next_token_ids_gpu,
|
||||
]]
|
||||
ranks = _get_ranks(
|
||||
logprobs[query_indices_gpu],
|
||||
next_token_ids_gpu,
|
||||
)
|
||||
assert selected_logprobs.shape[0] == ranks.shape[0]
|
||||
|
||||
batched_ranks_query_result = _get_ranks(
|
||||
logprobs[batched_logprobs_query_seq_indices_gpu],
|
||||
batched_logprobs_query_token_indices_gpu)
|
||||
|
||||
# Batched query for logprobs of topk tokens
|
||||
# Logprobs of topk tokens for a batch of sequence groups.
|
||||
# (num_query_tokens_across_batch).
|
||||
if largest_num_logprobs > 0:
|
||||
top_logprobs, top_token_ids = torch.topk(logprobs,
|
||||
largest_num_logprobs,
|
||||
@@ -685,79 +783,136 @@ def _get_logprobs(
|
||||
else:
|
||||
top_logprobs, top_token_ids = None, None
|
||||
|
||||
batched_logprobs_query_result = batched_logprobs_query_result.cpu()
|
||||
batched_ranks_query_result = batched_ranks_query_result.cpu()
|
||||
selected_logprobs = selected_logprobs.cpu()
|
||||
ranks = ranks.cpu()
|
||||
|
||||
# Gather results
|
||||
result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
|
||||
result_sample_logprobs: List[SampleLogprobs] = []
|
||||
sample_idx = 0
|
||||
query_result_idx = 0
|
||||
for i, (seq_group, sample_result) in enumerate(
|
||||
zip(sampling_metadata.seq_groups, sample_results)):
|
||||
seq_ids, sampling_params = seq_group
|
||||
next_token_ids, parent_ids = sample_result
|
||||
# Find prompt/sample logprobs.
|
||||
prompt_logprobs_per_seq_group: List[Optional[PromptLogprobs]] = []
|
||||
sample_logprobs_per_seq_group: List[SampleLogprobs] = []
|
||||
top_logprob_idx = 0
|
||||
selected_logprobs_idx = 0
|
||||
|
||||
# Prompt logprobs
|
||||
if (i < sampling_metadata.num_prompts
|
||||
and sampling_params.prompt_logprobs is not None):
|
||||
num_logprobs = sampling_params.prompt_logprobs
|
||||
prompt_tokens = sampling_metadata.seq_data[
|
||||
seq_ids[0]].prompt_token_ids
|
||||
group_prompt_logprobs: PromptLogprobs = [None]
|
||||
for token_id in prompt_tokens[1:]:
|
||||
prompt_logprobs_dict = {
|
||||
token_id:
|
||||
(batched_logprobs_query_result[query_result_idx].item(),
|
||||
batched_ranks_query_result[query_result_idx].item())
|
||||
}
|
||||
if num_logprobs > 0:
|
||||
prompt_logprobs_dict.update(
|
||||
zip(
|
||||
top_token_ids[sample_idx, :num_logprobs].tolist(),
|
||||
zip(
|
||||
top_logprobs[
|
||||
sample_idx, :num_logprobs].tolist(),
|
||||
range(1, num_logprobs + 1))))
|
||||
group_prompt_logprobs.append({
|
||||
token_id: Logprob(*logprob_rank)
|
||||
for token_id, logprob_rank in prompt_logprobs_dict.items()
|
||||
})
|
||||
sample_idx += 1
|
||||
query_result_idx += 1
|
||||
result_prompt_logprobs.append(group_prompt_logprobs)
|
||||
else:
|
||||
result_prompt_logprobs.append(None)
|
||||
for seq_group, sample_result in zip(sampling_metadata.seq_groups,
|
||||
sample_results):
|
||||
(prompt_logprobs, top_logprob_idx,
|
||||
selected_logprobs_idx) = _get_prompt_logprob_if_needed(
|
||||
seq_group, selected_logprobs, ranks, top_token_ids, top_logprobs,
|
||||
selected_logprobs_idx, top_logprob_idx)
|
||||
prompt_logprobs_per_seq_group.append(prompt_logprobs)
|
||||
|
||||
# Sample logprobs
|
||||
num_logprobs = sampling_params.logprobs
|
||||
if num_logprobs is None:
|
||||
num_logprobs = 0
|
||||
group_sample_logprobs: SampleLogprobs = []
|
||||
for next_token_id, parent_id in zip(next_token_ids, parent_ids):
|
||||
sample_logprobs_dict = {
|
||||
next_token_id:
|
||||
(batched_logprobs_query_result[query_result_idx].item(),
|
||||
batched_ranks_query_result[query_result_idx].item())
|
||||
(sampled_logprobs, top_logprob_idx,
|
||||
selected_logprobs_idx) = _get_sampled_logprob_if_needed(
|
||||
seq_group, sample_result, selected_logprobs, ranks, top_token_ids,
|
||||
top_logprobs, selected_logprobs_idx, top_logprob_idx)
|
||||
sample_logprobs_per_seq_group.append(sampled_logprobs)
|
||||
|
||||
return prompt_logprobs_per_seq_group, sample_logprobs_per_seq_group
|
||||
|
||||
|
||||
def _get_prompt_logprob_if_needed(
|
||||
seq_group: SequenceGroupToSample,
|
||||
selected_logprobs: torch.Tensor,
|
||||
ranks: torch.Tensor,
|
||||
top_token_ids: torch.Tensor,
|
||||
top_logprobs: torch.Tensor,
|
||||
selected_logprobs_idx: int,
|
||||
top_logprob_idx: int,
|
||||
):
|
||||
"""Compute the prompt logprob from a sequence group if needed."""
|
||||
sampling_params = seq_group.sampling_params
|
||||
is_prompt = seq_group.is_prompt
|
||||
|
||||
# Find prompt logprobs
|
||||
prompt_logprobs: Optional[PromptLogprobs] = None
|
||||
if (is_prompt and sampling_params.prompt_logprobs is not None):
|
||||
prompt_logprobs = []
|
||||
num_logprobs = sampling_params.prompt_logprobs
|
||||
next_prompt_tokens = _get_next_prompt_tokens(seq_group)
|
||||
for token_id in next_prompt_tokens:
|
||||
# Calculate the prompt logprob of the real prompt tokens.
|
||||
# Use tuple here for performance (to use to_list()).
|
||||
# {token_id: (logprob, rank_from_vocab)}
|
||||
prompt_logprobs_dict: Dict[int, Tuple[float, int]] = {
|
||||
token_id: (selected_logprobs[selected_logprobs_idx].item(),
|
||||
ranks[selected_logprobs_idx].item())
|
||||
}
|
||||
query_result_idx += 1
|
||||
if num_logprobs >= 0:
|
||||
sample_logprobs_dict.update(
|
||||
|
||||
# Add top K prompt logprobs along with its rank.
|
||||
if num_logprobs > 0:
|
||||
prompt_logprobs_dict.update(
|
||||
zip(
|
||||
top_token_ids[sample_idx +
|
||||
top_token_ids[top_logprob_idx, :num_logprobs].tolist(),
|
||||
zip(
|
||||
top_logprobs[
|
||||
top_logprob_idx, :num_logprobs].tolist(),
|
||||
# This is ranks. Since top_logprob is sorted,
|
||||
# we can just use a range here.
|
||||
range(1, num_logprobs + 1))))
|
||||
prompt_logprobs.append({
|
||||
token_id: Logprob(*logprob_and_rank)
|
||||
for token_id, logprob_and_rank in prompt_logprobs_dict.items()
|
||||
})
|
||||
# + 1 to go to the next prompt token.
|
||||
top_logprob_idx += 1
|
||||
selected_logprobs_idx += 1
|
||||
return prompt_logprobs, top_logprob_idx, selected_logprobs_idx
|
||||
|
||||
|
||||
def _get_sampled_logprob_if_needed(
|
||||
seq_group: SequenceGroupToSample,
|
||||
sample_result: Tuple[List[int], List[int]],
|
||||
selected_logprobs: torch.Tensor,
|
||||
ranks: torch.Tensor,
|
||||
top_token_ids: torch.Tensor,
|
||||
top_logprobs: torch.Tensor,
|
||||
selected_logprobs_idx: int,
|
||||
top_logprob_idx: int,
|
||||
):
|
||||
"""Compute the sample logprob if needed."""
|
||||
seq_ids = seq_group.seq_ids
|
||||
num_logprobs = seq_group.sampling_params.logprobs
|
||||
if num_logprobs is None:
|
||||
num_logprobs = 0
|
||||
sampled_logprobs: SampleLogprobs = []
|
||||
next_token_ids, parent_seq_ids = sample_result
|
||||
|
||||
if seq_group.do_sample:
|
||||
assert len(next_token_ids) > 0
|
||||
for (next_token_id, parent_id) in zip(next_token_ids, parent_seq_ids):
|
||||
# Calculate the sample logprob of the real sampled tokens.
|
||||
# Use tuple here for performance (to use to_list()).
|
||||
# token_id: (logprob, rank_from_vocab)
|
||||
sampled_logprobs_dict: Dict[int, Tuple[float, int]] = {
|
||||
next_token_id:
|
||||
(selected_logprobs[selected_logprobs_idx].item(),
|
||||
ranks[selected_logprobs_idx].item())
|
||||
}
|
||||
# +1 to go to the next sampled token. Note that
|
||||
# selected_logprobs can contain duplicates unlike top_logprobs
|
||||
# when beam search is enabled.
|
||||
selected_logprobs_idx += 1
|
||||
|
||||
# Second, add top K logprobs along with its rank.
|
||||
if num_logprobs >= 0:
|
||||
sampled_logprobs_dict.update(
|
||||
zip(
|
||||
top_token_ids[top_logprob_idx +
|
||||
parent_id, :num_logprobs].tolist(),
|
||||
zip(
|
||||
top_logprobs[sample_idx +
|
||||
top_logprobs[top_logprob_idx +
|
||||
parent_id, :num_logprobs].tolist(),
|
||||
# This is rank. Since top_logprob is sorted, we
|
||||
# can just use a range here.
|
||||
range(1, num_logprobs + 1))))
|
||||
group_sample_logprobs.append({
|
||||
token_id: Logprob(*logprob_rank)
|
||||
for token_id, logprob_rank in sample_logprobs_dict.items()
|
||||
sampled_logprobs.append({
|
||||
token_id: Logprob(*logprob_and_rank)
|
||||
for token_id, logprob_and_rank in
|
||||
sampled_logprobs_dict.items()
|
||||
})
|
||||
result_sample_logprobs.append(group_sample_logprobs)
|
||||
sample_idx += len(seq_ids)
|
||||
|
||||
return result_prompt_logprobs, result_sample_logprobs
|
||||
# There are len(seq_ids) number of sampled tokens for the current
|
||||
# sequence group in top_logprobs. Jump to the next seq_group.
|
||||
top_logprob_idx += len(seq_ids)
|
||||
return sampled_logprobs, top_logprob_idx, selected_logprobs_idx
|
||||
|
||||
|
||||
def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
|
||||
@@ -832,7 +987,7 @@ def _build_sampler_output(
|
||||
group_sample_logprobs) in zip(sampling_metadata.seq_groups,
|
||||
sample_results, prompt_logprobs,
|
||||
sample_logprobs):
|
||||
seq_ids, _ = seq_group
|
||||
seq_ids = seq_group.seq_ids
|
||||
next_token_ids, parent_ids = sample_result
|
||||
seq_outputs = []
|
||||
for parent_id, next_token_id, logprobs in zip(parent_ids,
|
||||
@@ -854,3 +1009,36 @@ def _build_sampler_output(
|
||||
sampled_token_probs=sampled_token_probs,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
)
|
||||
|
||||
|
||||
def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[str]:
|
||||
"""Get a list of next prompt tokens to compute logprob from a
|
||||
given sequence group.
|
||||
|
||||
It is used to compute prompt logprob. Imagine you have logprob for each
|
||||
query token. Query token needs to know the next prompt token id to compute
|
||||
prompt logprob. This is a helper to obtain next prompt token ids.
|
||||
|
||||
This API has to be used only when the caller knows seq_group is in prefill
|
||||
stage.
|
||||
|
||||
Returns:
|
||||
A list of next prompt tokens to compute logprob.
|
||||
"""
|
||||
assert seq_group.is_prompt, (
|
||||
"Caller should ensure the sequence group is in a prefill stage.")
|
||||
seq_ids = seq_group.seq_ids
|
||||
subquery_len = seq_group.subquery_len
|
||||
assert subquery_len is not None
|
||||
# prompt has only 1 seq id.
|
||||
assert len(seq_ids) == 1
|
||||
seq_data = seq_group.seq_data[seq_ids[0]]
|
||||
computed_len = seq_data.get_num_computed_tokens()
|
||||
prompt_tokens = seq_data.prompt_token_ids
|
||||
# +1 because we are looking for a next prompt token.
|
||||
next_token_index_start = computed_len + 1
|
||||
next_token_index_end = min(computed_len + subquery_len + 1,
|
||||
len(prompt_tokens))
|
||||
next_prompt_tokens = prompt_tokens[
|
||||
next_token_index_start:next_token_index_end]
|
||||
return next_prompt_tokens
|
||||
|
||||
Reference in New Issue
Block a user