[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)
This commit is contained in:
@@ -21,8 +21,6 @@ class RejectionSampler(nn.Module):
|
||||
nontrivial latency.
|
||||
"""
|
||||
super().__init__()
|
||||
self.probs_dtype = torch.float32
|
||||
self.token_id_dtype = torch.int64
|
||||
self._strict_mode = strict_mode
|
||||
|
||||
# NOTE: A "bonus token" is accepted iff all proposal tokens are
|
||||
@@ -44,6 +42,14 @@ class RejectionSampler(nn.Module):
|
||||
dtype=torch.long,
|
||||
device=device)
|
||||
|
||||
@property
|
||||
def probs_dtype(self):
|
||||
return torch.float32
|
||||
|
||||
@property
|
||||
def token_id_dtype(self):
|
||||
return torch.int64
|
||||
|
||||
def forward(
|
||||
self,
|
||||
target_probs: torch.Tensor,
|
||||
|
||||
@@ -587,4 +587,4 @@ def _build_sampler_output(
|
||||
SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
|
||||
sampler_output.append(
|
||||
SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
|
||||
return sampler_output
|
||||
return SamplerOutput(outputs=sampler_output)
|
||||
|
||||
Reference in New Issue
Block a user