[watermarking] Dual-key gumbel-max watermarking for speculative decoding support (#56122)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Simon Veitner <sveitner@redhat.com> Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Simon Veitner
Tomas Ruiz
parent
1ee4be4dbc
commit
7ee8a6dd01
@@ -137,7 +137,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
|
||||
# sample_pos is the predicted token's position P. Sampling keys a draw
|
||||
# by the position before the sampled token, P-1.
|
||||
return gumbel_sample(
|
||||
sampled = gumbel_sample(
|
||||
logits,
|
||||
idx_map,
|
||||
self.temperature,
|
||||
@@ -149,6 +149,14 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
logits_cache_col=self._step_cols[step],
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits,
|
||||
sampled,
|
||||
idx_map,
|
||||
self.temperature,
|
||||
)
|
||||
return sampled
|
||||
|
||||
def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
if self._draft_topk is not None:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -10,6 +11,7 @@ from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
InputBatch,
|
||||
get_num_sampled_and_rejected,
|
||||
@@ -78,8 +80,13 @@ class RejectionSampler:
|
||||
sampler: Sampler,
|
||||
spec_config: SpeculativeConfig,
|
||||
device: torch.device,
|
||||
*,
|
||||
watermark_key: int | None = None,
|
||||
):
|
||||
self.sampler = sampler
|
||||
self.watermark_key = watermark_key
|
||||
if watermark_key is not None:
|
||||
assert isinstance(sampler, GPUWatermarkSampler)
|
||||
self.num_speculative_steps = spec_config.num_speculative_tokens
|
||||
self.enable_adaptive_verification = spec_config.enable_adaptive_verification
|
||||
rejection_sample_method = spec_config.rejection_sample_method
|
||||
@@ -141,6 +148,23 @@ class RejectionSampler:
|
||||
in ("raw_logits", "processed_logits"),
|
||||
)
|
||||
|
||||
def _watermarking_kwargs(
|
||||
self,
|
||||
draft_sampled: torch.Tensor,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
expanded_local_pos: torch.Tensor,
|
||||
) -> dict[str, Any]:
|
||||
if self.watermark_key is None:
|
||||
return {}
|
||||
assert isinstance(self.sampler, GPUWatermarkSampler)
|
||||
return {
|
||||
"contexts": self.sampler._get_contexts(
|
||||
expanded_idx_mapping, expanded_local_pos, draft_sampled
|
||||
),
|
||||
"watermarking": self.sampler.watermarking.gpu,
|
||||
"watermark_key": self.watermark_key,
|
||||
}
|
||||
|
||||
def _verify(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -177,6 +201,9 @@ class RejectionSampler:
|
||||
self.synthetic_conditional_rates,
|
||||
use_fp64=self.sampler.use_fp64_gumbel,
|
||||
use_block_verification=self.use_block_verification,
|
||||
**self._watermarking_kwargs(
|
||||
draft_sampled, expanded_idx_mapping, expanded_local_pos
|
||||
),
|
||||
)
|
||||
return processed_logits, sampled, num_sampled
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -693,6 +694,39 @@ def _rejection_kernel(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _seeded_resample_argmax(
|
||||
residual_logits,
|
||||
block,
|
||||
mask,
|
||||
resample_token_idx,
|
||||
expanded_idx_mapping_ptr,
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
vocab_size,
|
||||
USE_FP64: tl.constexpr,
|
||||
):
|
||||
return gumbel_block_argmax(
|
||||
residual_logits,
|
||||
block,
|
||||
mask,
|
||||
resample_token_idx,
|
||||
expanded_idx_mapping_ptr,
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
None, # logits_cache_ptr
|
||||
0, # logits_cache_stride_0
|
||||
0, # logits_cache_stride_1
|
||||
None, # logits_cache_col_ptr
|
||||
vocab_size,
|
||||
IS_DRAFTING=False,
|
||||
APPLY_TEMPERATURE=False,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["watermark_key_0", "watermark_key_1"])
|
||||
def _resample_kernel(
|
||||
# [num_reqs, num_blocks]
|
||||
resampled_local_argmax_ptr,
|
||||
@@ -727,11 +761,20 @@ def _resample_kernel(
|
||||
pos_ptr,
|
||||
# [num_logits]
|
||||
cumulative_log_p_ptr,
|
||||
# [num_logits, CONTEXT_WIDTH]
|
||||
contexts_ptr,
|
||||
contexts_stride,
|
||||
# [max_num_reqs], uint8 view of a bool tensor
|
||||
watermarking_ptr,
|
||||
watermark_key_0,
|
||||
watermark_key_1,
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
USE_BLOCK_VERIFICATION: tl.constexpr,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
WATERMARK: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
resample_idx = tl.load(rejected_step_ptr + req_idx)
|
||||
@@ -763,10 +806,7 @@ def _resample_kernel(
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
|
||||
# Compute the residual logits to resample the rejected token from.
|
||||
if is_bonus or not is_valid_rejected_draft:
|
||||
# Bonus token (no rejections) or -1 placeholder token. In either case,
|
||||
# directly use the target logits.
|
||||
residual_logits = target_logits
|
||||
elif HAS_DRAFT_LOGITS:
|
||||
# draft_logits is stored pre-temperature, so apply scale first.
|
||||
@@ -796,12 +836,7 @@ def _resample_kernel(
|
||||
)
|
||||
target_log_probs += log_p_tau
|
||||
draft_log_probs = draft_logits - draft_lse
|
||||
# Compute the residual:
|
||||
# r(x) = max(p(x) - q(x), 0)
|
||||
# Gumbel sampling needs logits, so we compute it in log space:
|
||||
# log(r(x)) = log(max(exp(log_p(x)) - exp(log_q(x)), 0))
|
||||
# The more numerically stable form is:
|
||||
# log(max(exp(a) - exp(b), 0)) = a + log(max(1 - exp(b - a), 0))
|
||||
# Compute log(max(p - q, 0)) without subtracting probabilities.
|
||||
ratio = tl.exp(draft_log_probs - target_log_probs)
|
||||
residual_logits = tl.where(
|
||||
ratio < 1.0,
|
||||
@@ -809,38 +844,62 @@ def _resample_kernel(
|
||||
float("-inf"),
|
||||
).to(tl.float32)
|
||||
else:
|
||||
# One-hot draft. The residual is just the target distribution with
|
||||
# the rejected draft token probability zeroed out.
|
||||
# NOTE: During block verification, the residual becomes:
|
||||
# 0 if x == rejected_draft_token
|
||||
# p_tau * M_b(x) / Z otherwise
|
||||
# Therefore p_tau is a constant that cancels under normalization,
|
||||
# and does not need to be applied.
|
||||
# The block-verification factor is constant and cancels on normalization.
|
||||
residual_logits = tl.where(
|
||||
block != rejected_draft_token,
|
||||
target_logits,
|
||||
float("-inf"),
|
||||
).to(tl.float32)
|
||||
|
||||
# Resample the rejected/bonus token.
|
||||
value, idx = gumbel_block_argmax(
|
||||
residual_logits,
|
||||
block,
|
||||
mask,
|
||||
resample_token_idx,
|
||||
expanded_idx_mapping_ptr,
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
None, # logits_cache_ptr
|
||||
0, # logits_cache_stride_0
|
||||
0, # logits_cache_stride_1
|
||||
None, # logits_cache_col_ptr
|
||||
vocab_size,
|
||||
IS_DRAFTING=False,
|
||||
APPLY_TEMPERATURE=False,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
if WATERMARK:
|
||||
# Padded and greedy rows retain the stock draw.
|
||||
is_watermarked = (
|
||||
tl.load(
|
||||
watermarking_ptr + req_state_idx,
|
||||
mask=req_state_idx >= 0,
|
||||
other=0,
|
||||
)
|
||||
!= 0
|
||||
)
|
||||
if is_watermarked & (temp != 0.0):
|
||||
watermark_value, idx = philox_gumbel_block_argmax(
|
||||
residual_logits,
|
||||
mask,
|
||||
block_idx,
|
||||
contexts_ptr + resample_token_idx * contexts_stride,
|
||||
watermark_key_0.to(tl.uint32),
|
||||
watermark_key_1.to(tl.uint32),
|
||||
CONTEXT_WIDTH,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
# Detector compatibility fixes the keyed draw at fp32.
|
||||
value = watermark_value.to(tl.float64) if USE_FP64 else watermark_value
|
||||
else:
|
||||
value, idx = _seeded_resample_argmax(
|
||||
residual_logits,
|
||||
block,
|
||||
mask,
|
||||
resample_token_idx,
|
||||
expanded_idx_mapping_ptr,
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
vocab_size,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
else:
|
||||
value, idx = _seeded_resample_argmax(
|
||||
residual_logits,
|
||||
block,
|
||||
mask,
|
||||
resample_token_idx,
|
||||
expanded_idx_mapping_ptr,
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
vocab_size,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
token_id = block_idx * BLOCK_SIZE + idx
|
||||
tl.store(
|
||||
resampled_local_argmax_ptr
|
||||
@@ -947,6 +1006,9 @@ def rejection_sample(
|
||||
synthetic_conditional_rates: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
use_block_verification: bool = False,
|
||||
contexts: torch.Tensor | None = None,
|
||||
watermarking: torch.Tensor | None = None,
|
||||
watermark_key: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert target_logits.ndim == 2 and target_logits.stride(-1) == 1
|
||||
assert draft_logits is None or (
|
||||
@@ -954,6 +1016,30 @@ def rejection_sample(
|
||||
)
|
||||
num_reqs = cu_num_logits.shape[0] - 1
|
||||
num_logits, vocab_size = target_logits.shape
|
||||
|
||||
watermark = contexts is not None
|
||||
assert watermark == (watermarking is not None) == (watermark_key is not None), (
|
||||
"contexts, watermarking and watermark_key must be set together."
|
||||
)
|
||||
contexts_stride = 0
|
||||
context_width = 1
|
||||
watermarking_bytes: torch.Tensor | None = None
|
||||
watermark_key_0 = 0
|
||||
watermark_key_1 = 0
|
||||
if contexts is not None:
|
||||
assert watermarking is not None and watermark_key is not None
|
||||
assert contexts.ndim == 2 and contexts.shape[0] == num_logits
|
||||
# Context words are hashed as uint32, so int32 (the request-state token
|
||||
# dtype) and int64 both land on the same PRF stream, -1 included.
|
||||
assert contexts.dtype in (torch.int32, torch.int64)
|
||||
if contexts.stride(-1) != 1:
|
||||
contexts = contexts.contiguous()
|
||||
assert watermarking.ndim == 1 and watermarking.dtype == torch.bool
|
||||
contexts_stride = contexts.stride(0)
|
||||
context_width = contexts.shape[-1]
|
||||
watermarking_bytes = watermarking.view(torch.uint8)
|
||||
watermark_key_0 = watermark_key & 0xFFFFFFFF
|
||||
watermark_key_1 = watermark_key >> 32
|
||||
draft_logits_stride_0 = 0
|
||||
draft_logits_stride_1 = 0
|
||||
if has_draft_logits := draft_logits is not None:
|
||||
@@ -1165,11 +1251,18 @@ def rejection_sample(
|
||||
seed,
|
||||
pos,
|
||||
cumulative_log_p,
|
||||
contexts,
|
||||
contexts_stride,
|
||||
watermarking_bytes,
|
||||
watermark_key_0,
|
||||
watermark_key_1,
|
||||
vocab_size,
|
||||
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
|
||||
HAS_DRAFT_LOGITS=has_draft_logits,
|
||||
USE_FP64=use_fp64,
|
||||
USE_BLOCK_VERIFICATION=use_block_verification,
|
||||
CONTEXT_WIDTH=context_width,
|
||||
WATERMARK=watermark,
|
||||
)
|
||||
|
||||
# Insert the resampled tokens into the output sampled.
|
||||
|
||||
@@ -17,6 +17,11 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.models import supports_multimodal_embeddings
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.spec_decode import (
|
||||
DraftWatermarker,
|
||||
create_speculative_draft_watermarker,
|
||||
)
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
init_attn_backend,
|
||||
@@ -168,6 +173,17 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.draft_watermarker: DraftWatermarker | None = None
|
||||
watermark_config = getattr(vllm_config, "watermark_config", None)
|
||||
if watermark_config is not None:
|
||||
watermarker = create_watermarker(watermark_config)
|
||||
self.draft_watermarker = create_speculative_draft_watermarker(
|
||||
watermarker,
|
||||
self.max_num_reqs,
|
||||
device,
|
||||
watermark_config.allow_target_only_watermarking,
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = False
|
||||
self.pcp_manager: PCPManager | None = None
|
||||
|
||||
@@ -373,7 +389,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
) -> torch.Tensor:
|
||||
if draft_logits is not None:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
return gumbel_sample(
|
||||
sampled = gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
@@ -385,8 +401,23 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
logits_cache_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
if self.draft_watermarker is not None:
|
||||
sampled = self.draft_watermarker.sample(
|
||||
logits,
|
||||
sampled,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
)
|
||||
return sampled
|
||||
return self._greedy_sample_draft(hidden_states)
|
||||
|
||||
def prepare_watermarking(
|
||||
self, contexts: torch.Tensor, watermarking: torch.Tensor
|
||||
) -> None:
|
||||
if self.draft_watermarker is None:
|
||||
return
|
||||
self.draft_watermarker.prepare(contexts, watermarking)
|
||||
|
||||
def _copy_request_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
|
||||
Reference in New Issue
Block a user