[watermarking] add context deduplication support to speculative decoding (#56807)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Simon Veitner
Lucas Wilkinson
parent
73a5831127
commit
0f8b398158
@@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
self.sampler, input_batch.idx_mapping
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.propose(
|
||||
@@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
self.sampler, input_batch.idx_mapping
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
draft_tokens = self.speculator.propose(
|
||||
|
||||
Reference in New Issue
Block a user