[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)

This commit is contained in:
Cade Daniel
2024-03-08 23:32:46 -08:00
committed by GitHub
parent f48c6791b7
commit 8437bae6ef
21 changed files with 2786 additions and 215 deletions
+1 -1
View File
@@ -587,4 +587,4 @@ def _build_sampler_output(
SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
sampler_output.append(
SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
return sampler_output
return SamplerOutput(outputs=sampler_output)