Fix typical acceptance sampler with correct recovered token ids (#8562)
This commit is contained in:
@@ -80,7 +80,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
|
||||
target_probs = target_with_bonus_probs[:, :-1]
|
||||
accepted = self._evaluate_accepted_tokens(target_probs,
|
||||
draft_token_ids)
|
||||
recovered_token_ids = self._replacement_token_ids(target_probs)
|
||||
recovered_token_ids = self._get_recovered_token_ids(target_probs)
|
||||
output_token_ids = self._create_output(accepted, recovered_token_ids,
|
||||
draft_token_ids,
|
||||
bonus_token_ids)
|
||||
@@ -148,16 +148,10 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
|
||||
accepted_mask = candidates_prob > threshold
|
||||
return accepted_mask
|
||||
|
||||
def _replacement_token_ids(self, target_probs):
|
||||
def _get_recovered_token_ids(self, target_probs):
|
||||
"""
|
||||
Generate one replacement token ID for each sequence based on target
|
||||
probabilities. The replacement token is used as the fallback option
|
||||
if typical acceptance sampling does not accept any draft tokens for
|
||||
that particular sequence.
|
||||
|
||||
This method computes the token IDs to be replaced by selecting the
|
||||
token with the highest probability for each sequence in the first
|
||||
position. The rest of the output is filled with -1.
|
||||
The recovered token ids will fill the first unmatched token
|
||||
by the target token.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -168,13 +162,9 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor
|
||||
A tensor of shape (batch_size, k) with the replacement
|
||||
token IDs. Only the first column is set, and the rest of the
|
||||
columns are filled with -1.
|
||||
A tensor of shape (batch_size, k) with the recovered token
|
||||
ids which are selected from target probs.
|
||||
"""
|
||||
max_indices = torch.argmax(target_probs[:, 0, :], dim=1)
|
||||
output = -torch.ones((target_probs.shape[0], target_probs.shape[1]),
|
||||
dtype=self.token_id_dtype,
|
||||
device=target_probs.device)
|
||||
output[:, 0] = max_indices
|
||||
return output
|
||||
max_indices = torch.argmax(target_probs, dim=-1)
|
||||
|
||||
return max_indices
|
||||
|
||||
Reference in New Issue
Block a user