[watermarking] add context deduplication support to speculative decoding (#56807)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Raphaël Rialland
2026-09-30 12:22:10 -04:00
committed by GitHub
co-authored by Simon Veitner Lucas Wilkinson
parent 73a5831127
commit 0f8b398158
17 changed files with 1379 additions and 88 deletions
+2 -4
View File
@@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.propose(
@@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
self.sampler.watermarking.gpu[input_batch.idx_mapping],
self.sampler, input_batch.idx_mapping
)
with use_workspace_lane(self._draft_workspace_lane):
draft_tokens = self.speculator.propose(