[feature] Watermarked generation and detection (Gumbel-max algorithm) (#54053)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Raphaël Rialland
2026-09-09 21:00:15 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8c34a37232
commit ea40bb9e90
32 changed files with 1693 additions and 15 deletions
+21
View File
@@ -46,6 +46,27 @@ def test_optional_type():
assert optional_type_func("42") == 42
def test_watermark_config_cli():
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args(
[
"--model",
"dummy",
"--watermark-config",
'{"algorithm":"gumbel","key":42,"prf":"philox"}',
]
)
config = EngineArgs.from_cli_args(args).create_watermark_config()
assert config is not None
assert config.algorithm == "gumbel"
assert config.key == 42
assert config.context_width == 4
assert config.prf == "philox"
assert not config.supports_speculative_decoding
@pytest.mark.parametrize(
"options",
[
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
def test_chat_request_disables_watermarking():
request = ChatCompletionRequest(
messages=[{"role": "user", "content": "hello"}], watermarking=False
)
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
assert not params.watermarking
def test_completion_request_disables_watermarking():
request = CompletionRequest(prompt="hello", watermarking=False)
params = request.to_sampling_params(max_tokens=1)
assert not params.watermarking
def test_responses_request_disables_watermarking():
request = ResponsesRequest(input="hello", watermarking=False)
params = request.to_sampling_params(default_max_tokens=1)
assert not params.watermarking
@@ -120,17 +120,24 @@ class TestRedactSensitiveArgs:
API_KEY = "sk-test-secret-12345"
def test_redact_replaces_sensitive_values_only(self):
args = {"api_key": self.API_KEY, "hf_token": "hf_secret", "other": "visible"}
args = {
"api_key": self.API_KEY,
"hf_token": "hf_secret",
"watermark_config": {"algorithm": "gumbel", "key": 42},
"other": "visible",
}
redacted = redact_sensitive_args(args)
assert redacted == {
"api_key": "***",
"hf_token": "***",
"watermark_config": "***",
"other": "visible",
}
# original dict must not be mutated
assert args == {
"api_key": self.API_KEY,
"hf_token": "hf_secret",
"watermark_config": {"algorithm": "gumbel", "key": 42},
"other": "visible",
}
+61
View File
@@ -32,6 +32,7 @@ from vllm.config import (
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
WatermarkConfig,
update_config,
)
from vllm.config.compilation import CompilationMode, CUDAGraphMode
@@ -2574,6 +2575,66 @@ def test_draft_sample_method_gumbel_is_rejected():
)
def _watermarked_vllm_config() -> VllmConfig:
config = object.__new__(VllmConfig)
config.watermark_config = WatermarkConfig(key=42)
config.speculative_config = None
return config
def test_gumbel_watermark_rejects_speculative_decoding():
config = _watermarked_vllm_config()
config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=1,
)
with pytest.raises(ValueError, match="does not support speculative decoding"):
config._check_watermarking_unsupported()
def test_gumbel_watermark_rejects_beam_search():
with pytest.raises(ValueError, match="Beam search is not supported"):
_watermarked_vllm_config()._check_watermarking_unsupported(beam_search=True)
def test_gumbel_watermark_rejects_custom_sampler():
with pytest.raises(ValueError, match="custom samplers are not supported"):
_watermarked_vllm_config()._check_watermarking_unsupported(custom_sampler=True)
def test_watermark_key_must_fit_in_64_bits():
with pytest.raises(ValueError, match="64 bits"):
WatermarkConfig(key=2**64)
def test_unknown_watermark_prf_is_rejected():
with pytest.raises(ValidationError):
pydantic.TypeAdapter(WatermarkConfig).validate_python(
{"key": 42, "prf": "unsupported"}
)
def test_watermark_key_is_excluded_from_serialization():
serialized = pydantic.TypeAdapter(WatermarkConfig).dump_python(
WatermarkConfig(key=42), mode="json"
)
assert "key" not in serialized
def test_watermarking_forces_model_runner_v2(monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
with patch("vllm.config.vllm.logger.info_once") as info_once:
assert _watermarked_vllm_config().use_v2_model_runner
info_once.assert_called_once_with(
"Watermarking requires Model Runner V2 and overrides "
"VLLM_USE_V2_MODEL_RUNNER=0."
)
@patch("vllm.config.speculative.ModelConfig")
def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
"""Regression test: MTP + runai_streamer should use model_weights (original
+91
View File
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.watermarking import GumbelWatermarkDetector
_NON_WATERMARKED_TOKEN_IDS = list(range(64))
_WATERMARKED_TOKEN_IDS = [
85,
56,
123,
77,
105,
62,
9,
104,
16,
98,
123,
22,
16,
35,
47,
127,
77,
67,
19,
2,
10,
8,
75,
22,
127,
30,
95,
9,
101,
14,
113,
122,
85,
66,
122,
50,
38,
119,
15,
58,
68,
62,
24,
126,
80,
126,
68,
83,
81,
126,
62,
31,
18,
81,
64,
47,
22,
70,
47,
109,
54,
21,
65,
38,
]
def test_detection_on_known_sequence():
detector = GumbelWatermarkDetector(key=42, context_width=4, prf="philox")
assert detector.detect(_WATERMARKED_TOKEN_IDS).is_watermarked
assert not detector.detect(_NON_WATERMARKED_TOKEN_IDS).is_watermarked
def test_detector_handles_empty_input():
detector = GumbelWatermarkDetector(key=42, context_width=4, prf="philox")
contexts, targets = detector._prepare_inputs([])
detection = detector.detect([])
assert contexts.shape == (0, 4)
assert targets.shape == (0,)
assert detection.score == 0
assert detection.p_value == 1
assert not detection.is_watermarked
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.watermarking import GumbelWatermarkDetector, GumbelWatermarker
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
def test_gamma_survival_integer_shape():
assert _gamma_survival_integer_shape(0.0, 4) == 1.0
assert _gamma_survival_integer_shape(2.0, 1) == pytest.approx(0.1353352832)
def test_detector_deduplicates_repeated_prf_inputs_by_default():
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 1, 1, 1, 1])
assert detection.num_scored_tokens == 2
def test_detector_can_score_repeated_prf_inputs():
detection = GumbelWatermarkDetector(
key=42, context_width=1, deduplicate_contexts=False
).detect([1, 1, 1, 1, 1])
assert detection.num_scored_tokens == 5
def test_detector_deduplicates_context_even_when_target_differs():
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 2, 1, 3])
assert detection.num_scored_tokens == 3
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("key", [1, 2**32, 42, 15726070495360670683])
@pytest.mark.parametrize("context_width", [1, 4, 16])
def test_fused_watermarker_matches_cpu(key: int, context_width: int):
torch.manual_seed(0)
contexts = torch.randint(0, 248320, (32, context_width), dtype=torch.int64)
logits = torch.randn(32, 8193)
watermarker = GumbelWatermarker(key, context_width)
expected = watermarker.sample(logits, contexts, lambda values: None).token_ids
actual = watermarker.sample(
logits.cuda(), contexts.cuda(), lambda values: None
).token_ids.cpu()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_fused_watermarker_handles_nan_logits():
contexts = torch.zeros((2, 4), dtype=torch.int64, device="cuda")
logits = torch.full((2, 1025), float("nan"), device="cuda")
watermarker = GumbelWatermarker(key=42, context_width=4)
token_ids = watermarker.sample(logits, contexts, lambda values: None).token_ids
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_fused_watermarker_handles_noncontiguous_inputs():
contexts = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")[
:, ::2
]
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
watermarker = GumbelWatermarker(key=42, context_width=4)
expected = watermarker.sample(
logits.contiguous(), contexts.contiguous(), lambda values: None
).token_ids
actual = watermarker.sample(logits, contexts, lambda values: None).token_ids
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+44
View File
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.watermarking.prfs import PhiloxPRF
def test_philox_compatibility_vector():
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
token_ids = torch.tensor([7, 8])
expected_mantissas = [[1661833, 10571167], [9742331, 13375724]]
expected = (
(torch.tensor(expected_mantissas, dtype=torch.float64) + 1) / (2**24 + 1)
).to(torch.float32)
assert torch.equal(PhiloxPRF(42).uniform(contexts, token_ids), expected)
def test_prf_pairs_contexts_with_target_tokens():
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
token_ids = torch.tensor([[7], [8]])
assert PhiloxPRF(42).uniform(contexts, token_ids).shape == (2, 1)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("key", [42, 15726070495360670683])
@pytest.mark.parametrize("context_width", [1, 4, 16])
def test_philox_accelerator_matches_cpu(key: int, context_width: int):
contexts = torch.arange(2 * context_width, dtype=torch.int64).reshape(
2, context_width
)
token_ids = torch.arange(1024, dtype=torch.int64)
prf = PhiloxPRF(key)
expected = prf.uniform(contexts, token_ids)
actual = prf.uniform(contexts.cuda(), token_ids.cuda()).cpu()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+157
View File
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm import SamplingParams
from vllm.config.watermarking import WatermarkConfig
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.watermarker import WatermarkSample
from vllm.v1.worker.gpu.sample.sampler import Sampler
@pytest.mark.parametrize("algorithm", ["gumbel"])
def test_watermarker_contract(algorithm: str):
watermarker = create_watermarker(
WatermarkConfig(algorithm=algorithm, key=42, context_width=4)
)
logits = torch.zeros(2, 128)
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
random_sample = lambda sample_logits: sample_logits.argmax(dim=-1)
first = watermarker.sample(logits, contexts, random_sample)
second = watermarker.sample(logits, contexts, random_sample)
assert first.token_ids.shape == (2,)
assert first.logits.shape == logits.shape
assert torch.equal(first.token_ids, second.token_ids)
def test_gumbel_config_warns_about_degenerate_generations(monkeypatch):
messages: list[str] = []
monkeypatch.setattr(
"vllm.config.watermarking.logger.warning_once",
lambda message, *, scope: messages.append(message),
)
WatermarkConfig(key=42)
assert messages == [
"Single-key Gumbel-max watermarking may increase the frequency of "
"degenerate generations, including repetition loops."
]
def test_large_context_width_warns_but_is_allowed():
config = WatermarkConfig(key=42, context_width=17)
with pytest.warns(UserWarning, match="reduce robustness to edits"):
watermarker = create_watermarker(config)
assert watermarker.context_width == 17
def test_sampling_params_can_disable_watermarking():
assert SamplingParams().watermarking
assert not SamplingParams.from_optional(watermarking=False).watermarking
def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch):
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarking = SimpleNamespace(np=np.ones(1, dtype=bool))
messages: list[str] = []
monkeypatch.setattr(Sampler, "add_request", lambda *args, **kwargs: None)
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.logger.warning_once", messages.append
)
sampler.add_request(0, 1, SamplingParams(temperature=0))
sampler.add_request(0, 1, SamplingParams(temperature=1))
sampler.add_request(0, 1, SamplingParams(temperature=0, watermarking=False))
assert messages == [
(
"Watermarking is enabled, but greedy decoding (temperature=0) cannot be "
"watermarked. This request will use ordinary greedy sampling."
)
]
def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
class StubWatermarker:
context_width = 1
def sample(self, logits, contexts, random_sample):
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = StubWatermarker()
sampler.watermarking = SimpleNamespace(
np=np.array([True, False]), gpu=torch.tensor([True, False])
)
sampler.sampling_states = SimpleNamespace(
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
)
sampler.use_fp64_gumbel = False
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
2, 1, dtype=torch.int64
)
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.gumbel_sample",
lambda *args, **kwargs: torch.tensor([3, 4]),
)
logits = torch.zeros(2, 8)
sampled, output_logits = sampler._sample_random(
logits,
torch.tensor([0, 1]),
np.array([0, 1]),
torch.zeros(2, dtype=torch.int64),
None,
None,
False,
)
assert torch.equal(sampled, torch.tensor([7, 4]))
assert torch.equal(output_logits[0], torch.full((8,), 10.0))
assert torch.equal(output_logits[1], logits[1])
def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
class StubWatermarker:
context_width = 1
def sample(self, logits, contexts, random_sample):
raise AssertionError("watermarker should not run for greedy requests")
sampler = object.__new__(GPUWatermarkSampler)
sampler.watermarker = StubWatermarker()
sampler.watermarking = SimpleNamespace(
np=np.array([True, True]), gpu=torch.tensor([True, True])
)
sampler.sampling_states = SimpleNamespace(
temperature=SimpleNamespace(np=np.zeros(2), gpu=torch.zeros(2)),
)
expected = (torch.tensor([3, 4]), torch.zeros(2, 8))
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.Sampler._sample_random",
lambda *args, **kwargs: expected,
)
actual = sampler._sample_random(
torch.zeros(2, 8),
torch.tensor([0, 1]),
np.array([0, 1]),
torch.zeros(2, dtype=torch.int64),
None,
None,
False,
)
assert actual is expected