[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Simon Veitner
Tomas Ruiz
parent
1ee4be4dbc
commit
7ee8a6dd01
@@ -79,6 +79,10 @@ from vllm.v1.outputs import (
|
||||
)
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.spec_decode import (
|
||||
create_speculative_target_watermarker,
|
||||
speculative_target_watermark_key,
|
||||
)
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp
|
||||
@@ -472,8 +476,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.vllm_config.watermark_config is None:
|
||||
self.sampler = Sampler(**sampler_kwargs)
|
||||
else:
|
||||
watermarker = create_watermarker(self.vllm_config.watermark_config)
|
||||
self.sampler = GPUWatermarkSampler(watermarker, **sampler_kwargs)
|
||||
wm_config = self.vllm_config.watermark_config
|
||||
watermarker = create_watermarker(wm_config)
|
||||
if self.speculative_config is not None:
|
||||
watermarker = create_speculative_target_watermarker(watermarker)
|
||||
self.sampler = GPUWatermarkSampler(
|
||||
watermarker,
|
||||
deduplicate_contexts=wm_config.deduplicate_contexts,
|
||||
deduplicate_contexts_max_history=(
|
||||
wm_config.deduplicate_contexts_max_history
|
||||
),
|
||||
**sampler_kwargs,
|
||||
)
|
||||
custom = self.model_state.custom_sampler(self.sampler)
|
||||
|
||||
if custom:
|
||||
@@ -484,6 +498,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.sampler,
|
||||
self.speculative_config,
|
||||
self.device,
|
||||
watermark_key=speculative_target_watermark_key(
|
||||
self.vllm_config.watermark_config
|
||||
),
|
||||
)
|
||||
self.prompt_logprobs_worker = PromptLogprobsWorker(
|
||||
self.max_num_reqs,
|
||||
@@ -860,6 +877,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.propose(
|
||||
input_batch=input_batch,
|
||||
@@ -2086,6 +2108,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
self.sampler.watermarking.gpu[input_batch.idx_mapping],
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
draft_tokens = self.speculator.propose(
|
||||
input_batch,
|
||||
|
||||
Reference in New Issue
Block a user