[Misc] Misc code simplifications (#26450)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -147,22 +147,20 @@ class RejectionSampler(nn.Module):
|
||||
sampling_metadata: SamplingMetadata,
|
||||
metadata: SpecDecodeMetadata,
|
||||
) -> torch.Tensor:
|
||||
has_penalties = not sampling_metadata.no_penalties
|
||||
any_penalties_or_bad_words = (
|
||||
sampling_metadata.bad_words_token_ids or not sampling_metadata.no_penalties
|
||||
sampling_metadata.bad_words_token_ids or has_penalties
|
||||
)
|
||||
|
||||
output_token_ids = sampling_metadata.output_token_ids
|
||||
if any_penalties_or_bad_words:
|
||||
output_token_ids = self._combine_outputs_with_spec_tokens(
|
||||
sampling_metadata.output_token_ids,
|
||||
output_token_ids,
|
||||
sampling_metadata.spec_token_ids,
|
||||
)
|
||||
|
||||
# Calculate indices of target logits.
|
||||
if (
|
||||
sampling_metadata.allowed_token_ids_mask is not None
|
||||
or not sampling_metadata.no_penalties
|
||||
):
|
||||
if sampling_metadata.allowed_token_ids_mask is not None or has_penalties:
|
||||
num_requests = len(sampling_metadata.output_token_ids)
|
||||
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
|
||||
original_indices = torch.arange(num_requests, device="cpu")
|
||||
@@ -180,18 +178,15 @@ class RejectionSampler(nn.Module):
|
||||
logits.masked_fill_(token_mask, float("-inf"))
|
||||
|
||||
# Apply bad words exclusion.
|
||||
if sampling_metadata.bad_words_token_ids:
|
||||
if bad_words_token_ids := sampling_metadata.bad_words_token_ids:
|
||||
apply_bad_words_with_drafts(
|
||||
logits,
|
||||
sampling_metadata.bad_words_token_ids,
|
||||
output_token_ids,
|
||||
metadata.num_draft_tokens,
|
||||
logits, bad_words_token_ids, output_token_ids, metadata.num_draft_tokens
|
||||
)
|
||||
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
def apply_penalties(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
metadata: SpecDecodeMetadata,
|
||||
@@ -218,8 +213,8 @@ class RejectionSampler(nn.Module):
|
||||
)
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
def _combine_outputs_with_spec_tokens(
|
||||
self,
|
||||
output_token_ids: list[list[int]],
|
||||
spec_token_ids: Optional[list[list[int]]] = None,
|
||||
) -> list[list[int]]:
|
||||
|
||||
+26
-30
@@ -120,8 +120,8 @@ class Sampler(nn.Module):
|
||||
)
|
||||
return sampler_output
|
||||
|
||||
@staticmethod
|
||||
def apply_temperature(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
temp: torch.Tensor,
|
||||
all_random: bool,
|
||||
@@ -132,7 +132,8 @@ class Sampler(nn.Module):
|
||||
temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
|
||||
return logits.div_(temp.unsqueeze(dim=1))
|
||||
|
||||
def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
@staticmethod
|
||||
def greedy_sample(logits: torch.Tensor) -> torch.Tensor:
|
||||
return logits.argmax(dim=-1).view(-1)
|
||||
|
||||
def sample(
|
||||
@@ -191,11 +192,12 @@ class Sampler(nn.Module):
|
||||
)
|
||||
return sampled, processed_logprobs
|
||||
|
||||
def compute_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
@staticmethod
|
||||
def compute_logprobs(logits: torch.Tensor) -> torch.Tensor:
|
||||
return logits.log_softmax(dim=-1, dtype=torch.float32)
|
||||
|
||||
@staticmethod
|
||||
def gather_logprobs(
|
||||
self,
|
||||
logprobs: torch.Tensor,
|
||||
num_logprobs: int,
|
||||
token_ids: torch.Tensor,
|
||||
@@ -238,8 +240,8 @@ class Sampler(nn.Module):
|
||||
|
||||
return LogprobsTensors(indices, logprobs, token_ranks)
|
||||
|
||||
@staticmethod
|
||||
def _combine_outputs_with_spec_tokens(
|
||||
self,
|
||||
output_token_ids: list[list[int]],
|
||||
spec_token_ids: Optional[list[list[int]]] = None,
|
||||
) -> list[list[int]]:
|
||||
@@ -257,8 +259,9 @@ class Sampler(nn.Module):
|
||||
sampling_metadata: SamplingMetadata,
|
||||
predict_bonus_token: bool,
|
||||
) -> torch.Tensor:
|
||||
bad_words_token_ids = sampling_metadata.bad_words_token_ids
|
||||
any_penalties_or_bad_words = (
|
||||
sampling_metadata.bad_words_token_ids or not sampling_metadata.no_penalties
|
||||
bool(bad_words_token_ids) or not sampling_metadata.no_penalties
|
||||
)
|
||||
|
||||
output_token_ids = sampling_metadata.output_token_ids
|
||||
@@ -266,7 +269,7 @@ class Sampler(nn.Module):
|
||||
# Combine base outputs with spec tokens when speculative decoding
|
||||
# is enabled.
|
||||
output_token_ids = self._combine_outputs_with_spec_tokens(
|
||||
sampling_metadata.output_token_ids,
|
||||
output_token_ids,
|
||||
sampling_metadata.spec_token_ids,
|
||||
)
|
||||
|
||||
@@ -275,14 +278,8 @@ class Sampler(nn.Module):
|
||||
logits.masked_fill_(sampling_metadata.allowed_token_ids_mask, float("-inf"))
|
||||
|
||||
# Apply bad words exclusion.
|
||||
if sampling_metadata.bad_words_token_ids:
|
||||
apply_bad_words(
|
||||
logits,
|
||||
sampling_metadata.bad_words_token_ids,
|
||||
output_token_ids
|
||||
if output_token_ids is not None
|
||||
else sampling_metadata.output_token_ids,
|
||||
)
|
||||
if bad_words_token_ids:
|
||||
apply_bad_words(logits, bad_words_token_ids, output_token_ids)
|
||||
|
||||
# Apply logits processors which can impact greedy sampling.
|
||||
for processor in sampling_metadata.logitsprocs.non_argmax_invariant:
|
||||
@@ -292,22 +289,21 @@ class Sampler(nn.Module):
|
||||
logits = self.apply_penalties(logits, sampling_metadata, output_token_ids)
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
def apply_penalties(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
output_token_ids: Optional[list[list[int]]] = None,
|
||||
output_token_ids: list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
if not sampling_metadata.no_penalties:
|
||||
assert sampling_metadata.prompt_token_ids is not None
|
||||
logits = apply_all_penalties(
|
||||
logits,
|
||||
sampling_metadata.prompt_token_ids,
|
||||
sampling_metadata.presence_penalties,
|
||||
sampling_metadata.frequency_penalties,
|
||||
sampling_metadata.repetition_penalties,
|
||||
output_token_ids
|
||||
if output_token_ids is not None
|
||||
else sampling_metadata.output_token_ids,
|
||||
)
|
||||
return logits
|
||||
if sampling_metadata.no_penalties:
|
||||
return logits
|
||||
|
||||
assert sampling_metadata.prompt_token_ids is not None
|
||||
return apply_all_penalties(
|
||||
logits,
|
||||
sampling_metadata.prompt_token_ids,
|
||||
sampling_metadata.presence_penalties,
|
||||
sampling_metadata.frequency_penalties,
|
||||
sampling_metadata.repetition_penalties,
|
||||
output_token_ids,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user