[Speculative decoding] [Multi-Step] decouple should_modify_greedy_probs_inplace (#6971)

This commit is contained in:
William Lin
2024-08-09 05:42:45 +00:00
committed by GitHub
parent 99b4cf5f23
commit 57b7be0e1c
8 changed files with 52 additions and 3 deletions
+2 -2
View File
@@ -51,6 +51,7 @@ class Sampler(nn.Module):
# containing the sampled token ids and probabilities. This is used by
# speculative decoding.
self.include_gpu_probs_tensor = False
self.should_modify_greedy_probs_inplace = False
def _init_sampling_tensors(
self,
@@ -177,8 +178,7 @@ class Sampler(nn.Module):
This is used by speculative decoding, which requires that the sampling
method be encoded into the probability distribution.
"""
# Modify greedy probs if include_gpu_probs_tensor is set.
return self.include_gpu_probs_tensor
return self.should_modify_greedy_probs_inplace
def _get_bin_counts_and_mask(