[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Simon Veitner <sveitner@redhat.com>
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Raphaël Rialland
2026-09-12 18:42:01 +00:00
committed by GitHub
co-authored by OpenAI Codex Simon Veitner Tomas Ruiz
parent 1ee4be4dbc
commit 7ee8a6dd01
28 changed files with 3460 additions and 245 deletions
@@ -137,7 +137,7 @@ class DSparkSpeculator(DFlashSpeculator):
# sample_pos is the predicted token's position P. Sampling keys a draw
# by the position before the sampled token, P-1.
return gumbel_sample(
sampled = gumbel_sample(
logits,
idx_map,
self.temperature,
@@ -149,6 +149,14 @@ class DSparkSpeculator(DFlashSpeculator):
logits_cache_col=self._step_cols[step],
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits,
sampled,
idx_map,
self.temperature,
)
return sampled
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
if self._draft_topk is not None:
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Iterator
from typing import Any
import numpy as np
import torch
@@ -10,6 +11,7 @@ from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
get_num_sampled_and_rejected,
@@ -78,8 +80,13 @@ class RejectionSampler:
sampler: Sampler,
spec_config: SpeculativeConfig,
device: torch.device,
*,
watermark_key: int | None = None,
):
self.sampler = sampler
self.watermark_key = watermark_key
if watermark_key is not None:
assert isinstance(sampler, GPUWatermarkSampler)
self.num_speculative_steps = spec_config.num_speculative_tokens
self.enable_adaptive_verification = spec_config.enable_adaptive_verification
rejection_sample_method = spec_config.rejection_sample_method
@@ -141,6 +148,23 @@ class RejectionSampler:
in ("raw_logits", "processed_logits"),
)
def _watermarking_kwargs(
self,
draft_sampled: torch.Tensor,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
) -> dict[str, Any]:
if self.watermark_key is None:
return {}
assert isinstance(self.sampler, GPUWatermarkSampler)
return {
"contexts": self.sampler._get_contexts(
expanded_idx_mapping, expanded_local_pos, draft_sampled
),
"watermarking": self.sampler.watermarking.gpu,
"watermark_key": self.watermark_key,
}
def _verify(
self,
logits: torch.Tensor,
@@ -177,6 +201,9 @@ class RejectionSampler:
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
use_block_verification=self.use_block_verification,
**self._watermarking_kwargs(
draft_sampled, expanded_idx_mapping, expanded_local_pos
),
)
return processed_logits, sampled, num_sampled
@@ -4,6 +4,7 @@ import torch
from vllm.triton_utils import tl, tldevice, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
@triton.jit
@@ -693,6 +694,39 @@ def _rejection_kernel(
@triton.jit
def _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64: tl.constexpr,
):
return gumbel_block_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
None, # logits_cache_ptr
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
)
@triton.jit(do_not_specialize=["watermark_key_0", "watermark_key_1"])
def _resample_kernel(
# [num_reqs, num_blocks]
resampled_local_argmax_ptr,
@@ -727,11 +761,20 @@ def _resample_kernel(
pos_ptr,
# [num_logits]
cumulative_log_p_ptr,
# [num_logits, CONTEXT_WIDTH]
contexts_ptr,
contexts_stride,
# [max_num_reqs], uint8 view of a bool tensor
watermarking_ptr,
watermark_key_0,
watermark_key_1,
vocab_size,
BLOCK_SIZE: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
USE_FP64: tl.constexpr,
USE_BLOCK_VERIFICATION: tl.constexpr,
CONTEXT_WIDTH: tl.constexpr,
WATERMARK: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -763,10 +806,7 @@ def _resample_kernel(
other=float("-inf"),
).to(tl.float32)
# Compute the residual logits to resample the rejected token from.
if is_bonus or not is_valid_rejected_draft:
# Bonus token (no rejections) or -1 placeholder token. In either case,
# directly use the target logits.
residual_logits = target_logits
elif HAS_DRAFT_LOGITS:
# draft_logits is stored pre-temperature, so apply scale first.
@@ -796,12 +836,7 @@ def _resample_kernel(
)
target_log_probs += log_p_tau
draft_log_probs = draft_logits - draft_lse
# Compute the residual:
# r(x) = max(p(x) - q(x), 0)
# Gumbel sampling needs logits, so we compute it in log space:
# log(r(x)) = log(max(exp(log_p(x)) - exp(log_q(x)), 0))
# The more numerically stable form is:
# log(max(exp(a) - exp(b), 0)) = a + log(max(1 - exp(b - a), 0))
# Compute log(max(p - q, 0)) without subtracting probabilities.
ratio = tl.exp(draft_log_probs - target_log_probs)
residual_logits = tl.where(
ratio < 1.0,
@@ -809,38 +844,62 @@ def _resample_kernel(
float("-inf"),
).to(tl.float32)
else:
# One-hot draft. The residual is just the target distribution with
# the rejected draft token probability zeroed out.
# NOTE: During block verification, the residual becomes:
# 0 if x == rejected_draft_token
# p_tau * M_b(x) / Z otherwise
# Therefore p_tau is a constant that cancels under normalization,
# and does not need to be applied.
# The block-verification factor is constant and cancels on normalization.
residual_logits = tl.where(
block != rejected_draft_token,
target_logits,
float("-inf"),
).to(tl.float32)
# Resample the rejected/bonus token.
value, idx = gumbel_block_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
None, # logits_cache_ptr
0, # logits_cache_stride_0
0, # logits_cache_stride_1
None, # logits_cache_col_ptr
vocab_size,
IS_DRAFTING=False,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
)
if WATERMARK:
# Padded and greedy rows retain the stock draw.
is_watermarked = (
tl.load(
watermarking_ptr + req_state_idx,
mask=req_state_idx >= 0,
other=0,
)
!= 0
)
if is_watermarked & (temp != 0.0):
watermark_value, idx = philox_gumbel_block_argmax(
residual_logits,
mask,
block_idx,
contexts_ptr + resample_token_idx * contexts_stride,
watermark_key_0.to(tl.uint32),
watermark_key_1.to(tl.uint32),
CONTEXT_WIDTH,
BLOCK_SIZE,
)
# Detector compatibility fixes the keyed draw at fp32.
value = watermark_value.to(tl.float64) if USE_FP64 else watermark_value
else:
value, idx = _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64=USE_FP64,
)
else:
value, idx = _seeded_resample_argmax(
residual_logits,
block,
mask,
resample_token_idx,
expanded_idx_mapping_ptr,
temp_ptr,
seed_ptr,
pos_ptr,
vocab_size,
USE_FP64=USE_FP64,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(
resampled_local_argmax_ptr
@@ -947,6 +1006,9 @@ def rejection_sample(
synthetic_conditional_rates: torch.Tensor | None = None,
use_fp64: bool = False,
use_block_verification: bool = False,
contexts: torch.Tensor | None = None,
watermarking: torch.Tensor | None = None,
watermark_key: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
assert draft_logits is None or (
@@ -954,6 +1016,30 @@ def rejection_sample(
)
num_reqs = cu_num_logits.shape[0] - 1
num_logits, vocab_size = target_logits.shape
watermark = contexts is not None
assert watermark == (watermarking is not None) == (watermark_key is not None), (
"contexts, watermarking and watermark_key must be set together."
)
contexts_stride = 0
context_width = 1
watermarking_bytes: torch.Tensor | None = None
watermark_key_0 = 0
watermark_key_1 = 0
if contexts is not None:
assert watermarking is not None and watermark_key is not None
assert contexts.ndim == 2 and contexts.shape[0] == num_logits
# Context words are hashed as uint32, so int32 (the request-state token
# dtype) and int64 both land on the same PRF stream, -1 included.
assert contexts.dtype in (torch.int32, torch.int64)
if contexts.stride(-1) != 1:
contexts = contexts.contiguous()
assert watermarking.ndim == 1 and watermarking.dtype == torch.bool
contexts_stride = contexts.stride(0)
context_width = contexts.shape[-1]
watermarking_bytes = watermarking.view(torch.uint8)
watermark_key_0 = watermark_key & 0xFFFFFFFF
watermark_key_1 = watermark_key >> 32
draft_logits_stride_0 = 0
draft_logits_stride_1 = 0
if has_draft_logits := draft_logits is not None:
@@ -1165,11 +1251,18 @@ def rejection_sample(
seed,
pos,
cumulative_log_p,
contexts,
contexts_stride,
watermarking_bytes,
watermark_key_0,
watermark_key_1,
vocab_size,
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
HAS_DRAFT_LOGITS=has_draft_logits,
USE_FP64=use_fp64,
USE_BLOCK_VERIFICATION=use_block_verification,
CONTEXT_WIDTH=context_width,
WATERMARK=watermark,
)
# Insert the resampled tokens into the output sampled.
+32 -1
View File
@@ -17,6 +17,11 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.spec_decode import (
DraftWatermarker,
create_speculative_draft_watermarker,
)
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
init_attn_backend,
@@ -168,6 +173,17 @@ class DraftModelSpeculator(BaseSpeculator):
device=device,
)
self.draft_watermarker: DraftWatermarker | None = None
watermark_config = getattr(vllm_config, "watermark_config", None)
if watermark_config is not None:
watermarker = create_watermarker(watermark_config)
self.draft_watermarker = create_speculative_draft_watermarker(
watermarker,
self.max_num_reqs,
device,
watermark_config.allow_target_only_watermarking,
)
self.supports_mm_inputs = False
self.pcp_manager: PCPManager | None = None
@@ -373,7 +389,7 @@ class DraftModelSpeculator(BaseSpeculator):
) -> torch.Tensor:
if draft_logits is not None:
logits = self.model.compute_logits(hidden_states)
return gumbel_sample(
sampled = gumbel_sample(
logits,
idx_mapping,
temperature,
@@ -385,8 +401,23 @@ class DraftModelSpeculator(BaseSpeculator):
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
if self.draft_watermarker is not None:
sampled = self.draft_watermarker.sample(
logits,
sampled,
idx_mapping,
temperature,
)
return sampled
return self._greedy_sample_draft(hidden_states)
def prepare_watermarking(
self, contexts: torch.Tensor, watermarking: torch.Tensor
) -> None:
if self.draft_watermarker is None:
return
self.draft_watermarker.prepare(contexts, watermarking)
def _copy_request_inputs(
self,
num_reqs: int,