[feature] Watermarked generation and detection (Gumbel-max algorithm) (#54053)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8c34a37232
commit
ea40bb9e90
@@ -46,6 +46,27 @@ def test_optional_type():
|
||||
assert optional_type_func("42") == 42
|
||||
|
||||
|
||||
def test_watermark_config_cli():
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--model",
|
||||
"dummy",
|
||||
"--watermark-config",
|
||||
'{"algorithm":"gumbel","key":42,"prf":"philox"}',
|
||||
]
|
||||
)
|
||||
|
||||
config = EngineArgs.from_cli_args(args).create_watermark_config()
|
||||
|
||||
assert config is not None
|
||||
assert config.algorithm == "gumbel"
|
||||
assert config.key == 42
|
||||
assert config.context_width == 4
|
||||
assert config.prf == "philox"
|
||||
assert not config.supports_speculative_decoding
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
def test_chat_request_disables_watermarking():
|
||||
request = ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "hello"}], watermarking=False
|
||||
)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert not params.watermarking
|
||||
|
||||
|
||||
def test_completion_request_disables_watermarking():
|
||||
request = CompletionRequest(prompt="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(max_tokens=1)
|
||||
|
||||
assert not params.watermarking
|
||||
|
||||
|
||||
def test_responses_request_disables_watermarking():
|
||||
request = ResponsesRequest(input="hello", watermarking=False)
|
||||
|
||||
params = request.to_sampling_params(default_max_tokens=1)
|
||||
|
||||
assert not params.watermarking
|
||||
@@ -120,17 +120,24 @@ class TestRedactSensitiveArgs:
|
||||
API_KEY = "sk-test-secret-12345"
|
||||
|
||||
def test_redact_replaces_sensitive_values_only(self):
|
||||
args = {"api_key": self.API_KEY, "hf_token": "hf_secret", "other": "visible"}
|
||||
args = {
|
||||
"api_key": self.API_KEY,
|
||||
"hf_token": "hf_secret",
|
||||
"watermark_config": {"algorithm": "gumbel", "key": 42},
|
||||
"other": "visible",
|
||||
}
|
||||
redacted = redact_sensitive_args(args)
|
||||
assert redacted == {
|
||||
"api_key": "***",
|
||||
"hf_token": "***",
|
||||
"watermark_config": "***",
|
||||
"other": "visible",
|
||||
}
|
||||
# original dict must not be mutated
|
||||
assert args == {
|
||||
"api_key": self.API_KEY,
|
||||
"hf_token": "hf_secret",
|
||||
"watermark_config": {"algorithm": "gumbel", "key": 42},
|
||||
"other": "visible",
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.config import (
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
WatermarkConfig,
|
||||
update_config,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
@@ -2574,6 +2575,66 @@ def test_draft_sample_method_gumbel_is_rejected():
|
||||
)
|
||||
|
||||
|
||||
def _watermarked_vllm_config() -> VllmConfig:
|
||||
config = object.__new__(VllmConfig)
|
||||
config.watermark_config = WatermarkConfig(key=42)
|
||||
config.speculative_config = None
|
||||
return config
|
||||
|
||||
|
||||
def test_gumbel_watermark_rejects_speculative_decoding():
|
||||
config = _watermarked_vllm_config()
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support speculative decoding"):
|
||||
config._check_watermarking_unsupported()
|
||||
|
||||
|
||||
def test_gumbel_watermark_rejects_beam_search():
|
||||
with pytest.raises(ValueError, match="Beam search is not supported"):
|
||||
_watermarked_vllm_config()._check_watermarking_unsupported(beam_search=True)
|
||||
|
||||
|
||||
def test_gumbel_watermark_rejects_custom_sampler():
|
||||
with pytest.raises(ValueError, match="custom samplers are not supported"):
|
||||
_watermarked_vllm_config()._check_watermarking_unsupported(custom_sampler=True)
|
||||
|
||||
|
||||
def test_watermark_key_must_fit_in_64_bits():
|
||||
with pytest.raises(ValueError, match="64 bits"):
|
||||
WatermarkConfig(key=2**64)
|
||||
|
||||
|
||||
def test_unknown_watermark_prf_is_rejected():
|
||||
with pytest.raises(ValidationError):
|
||||
pydantic.TypeAdapter(WatermarkConfig).validate_python(
|
||||
{"key": 42, "prf": "unsupported"}
|
||||
)
|
||||
|
||||
|
||||
def test_watermark_key_is_excluded_from_serialization():
|
||||
serialized = pydantic.TypeAdapter(WatermarkConfig).dump_python(
|
||||
WatermarkConfig(key=42), mode="json"
|
||||
)
|
||||
|
||||
assert "key" not in serialized
|
||||
|
||||
|
||||
def test_watermarking_forces_model_runner_v2(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
|
||||
with patch("vllm.config.vllm.logger.info_once") as info_once:
|
||||
assert _watermarked_vllm_config().use_v2_model_runner
|
||||
|
||||
info_once.assert_called_once_with(
|
||||
"Watermarking requires Model Runner V2 and overrides "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=0."
|
||||
)
|
||||
|
||||
|
||||
@patch("vllm.config.speculative.ModelConfig")
|
||||
def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls):
|
||||
"""Regression test: MTP + runai_streamer should use model_weights (original
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.v1.watermarking import GumbelWatermarkDetector
|
||||
|
||||
_NON_WATERMARKED_TOKEN_IDS = list(range(64))
|
||||
_WATERMARKED_TOKEN_IDS = [
|
||||
85,
|
||||
56,
|
||||
123,
|
||||
77,
|
||||
105,
|
||||
62,
|
||||
9,
|
||||
104,
|
||||
16,
|
||||
98,
|
||||
123,
|
||||
22,
|
||||
16,
|
||||
35,
|
||||
47,
|
||||
127,
|
||||
77,
|
||||
67,
|
||||
19,
|
||||
2,
|
||||
10,
|
||||
8,
|
||||
75,
|
||||
22,
|
||||
127,
|
||||
30,
|
||||
95,
|
||||
9,
|
||||
101,
|
||||
14,
|
||||
113,
|
||||
122,
|
||||
85,
|
||||
66,
|
||||
122,
|
||||
50,
|
||||
38,
|
||||
119,
|
||||
15,
|
||||
58,
|
||||
68,
|
||||
62,
|
||||
24,
|
||||
126,
|
||||
80,
|
||||
126,
|
||||
68,
|
||||
83,
|
||||
81,
|
||||
126,
|
||||
62,
|
||||
31,
|
||||
18,
|
||||
81,
|
||||
64,
|
||||
47,
|
||||
22,
|
||||
70,
|
||||
47,
|
||||
109,
|
||||
54,
|
||||
21,
|
||||
65,
|
||||
38,
|
||||
]
|
||||
|
||||
|
||||
def test_detection_on_known_sequence():
|
||||
detector = GumbelWatermarkDetector(key=42, context_width=4, prf="philox")
|
||||
|
||||
assert detector.detect(_WATERMARKED_TOKEN_IDS).is_watermarked
|
||||
assert not detector.detect(_NON_WATERMARKED_TOKEN_IDS).is_watermarked
|
||||
|
||||
|
||||
def test_detector_handles_empty_input():
|
||||
detector = GumbelWatermarkDetector(key=42, context_width=4, prf="philox")
|
||||
contexts, targets = detector._prepare_inputs([])
|
||||
detection = detector.detect([])
|
||||
|
||||
assert contexts.shape == (0, 4)
|
||||
assert targets.shape == (0,)
|
||||
assert detection.score == 0
|
||||
assert detection.p_value == 1
|
||||
assert not detection.is_watermarked
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking import GumbelWatermarkDetector, GumbelWatermarker
|
||||
from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape
|
||||
|
||||
|
||||
def test_gamma_survival_integer_shape():
|
||||
assert _gamma_survival_integer_shape(0.0, 4) == 1.0
|
||||
assert _gamma_survival_integer_shape(2.0, 1) == pytest.approx(0.1353352832)
|
||||
|
||||
|
||||
def test_detector_deduplicates_repeated_prf_inputs_by_default():
|
||||
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 1, 1, 1, 1])
|
||||
|
||||
assert detection.num_scored_tokens == 2
|
||||
|
||||
|
||||
def test_detector_can_score_repeated_prf_inputs():
|
||||
detection = GumbelWatermarkDetector(
|
||||
key=42, context_width=1, deduplicate_contexts=False
|
||||
).detect([1, 1, 1, 1, 1])
|
||||
|
||||
assert detection.num_scored_tokens == 5
|
||||
|
||||
|
||||
def test_detector_deduplicates_context_even_when_target_differs():
|
||||
detection = GumbelWatermarkDetector(key=42, context_width=1).detect([1, 2, 1, 3])
|
||||
|
||||
assert detection.num_scored_tokens == 3
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("key", [1, 2**32, 42, 15726070495360670683])
|
||||
@pytest.mark.parametrize("context_width", [1, 4, 16])
|
||||
def test_fused_watermarker_matches_cpu(key: int, context_width: int):
|
||||
torch.manual_seed(0)
|
||||
contexts = torch.randint(0, 248320, (32, context_width), dtype=torch.int64)
|
||||
logits = torch.randn(32, 8193)
|
||||
watermarker = GumbelWatermarker(key, context_width)
|
||||
|
||||
expected = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
actual = watermarker.sample(
|
||||
logits.cuda(), contexts.cuda(), lambda values: None
|
||||
).token_ids.cpu()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_fused_watermarker_handles_nan_logits():
|
||||
contexts = torch.zeros((2, 4), dtype=torch.int64, device="cuda")
|
||||
logits = torch.full((2, 1025), float("nan"), device="cuda")
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
token_ids = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
|
||||
assert torch.all((token_ids >= 0) & (token_ids < logits.shape[-1]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_fused_watermarker_handles_noncontiguous_inputs():
|
||||
contexts = torch.randint(0, 248320, (8, 8), dtype=torch.int64, device="cuda")[
|
||||
:, ::2
|
||||
]
|
||||
logits = torch.randn(8, 2050, device="cuda")[:, ::2]
|
||||
watermarker = GumbelWatermarker(key=42, context_width=4)
|
||||
|
||||
expected = watermarker.sample(
|
||||
logits.contiguous(), contexts.contiguous(), lambda values: None
|
||||
).token_ids
|
||||
actual = watermarker.sample(logits, contexts, lambda values: None).token_ids
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF
|
||||
|
||||
|
||||
def test_philox_compatibility_vector():
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
token_ids = torch.tensor([7, 8])
|
||||
expected_mantissas = [[1661833, 10571167], [9742331, 13375724]]
|
||||
expected = (
|
||||
(torch.tensor(expected_mantissas, dtype=torch.float64) + 1) / (2**24 + 1)
|
||||
).to(torch.float32)
|
||||
|
||||
assert torch.equal(PhiloxPRF(42).uniform(contexts, token_ids), expected)
|
||||
|
||||
|
||||
def test_prf_pairs_contexts_with_target_tokens():
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
token_ids = torch.tensor([[7], [8]])
|
||||
|
||||
assert PhiloxPRF(42).uniform(contexts, token_ids).shape == (2, 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
@pytest.mark.parametrize("key", [42, 15726070495360670683])
|
||||
@pytest.mark.parametrize("context_width", [1, 4, 16])
|
||||
def test_philox_accelerator_matches_cpu(key: int, context_width: int):
|
||||
contexts = torch.arange(2 * context_width, dtype=torch.int64).reshape(
|
||||
2, context_width
|
||||
)
|
||||
token_ids = torch.arange(1024, dtype=torch.int64)
|
||||
prf = PhiloxPRF(key)
|
||||
|
||||
expected = prf.uniform(contexts, token_ids)
|
||||
actual = prf.uniform(contexts.cuda(), token_ids.cuda()).cpu()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.watermarker import WatermarkSample
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algorithm", ["gumbel"])
|
||||
def test_watermarker_contract(algorithm: str):
|
||||
watermarker = create_watermarker(
|
||||
WatermarkConfig(algorithm=algorithm, key=42, context_width=4)
|
||||
)
|
||||
logits = torch.zeros(2, 128)
|
||||
contexts = torch.tensor([[1, 2, 3, 4], [4, 5, 6, 7]])
|
||||
random_sample = lambda sample_logits: sample_logits.argmax(dim=-1)
|
||||
|
||||
first = watermarker.sample(logits, contexts, random_sample)
|
||||
second = watermarker.sample(logits, contexts, random_sample)
|
||||
|
||||
assert first.token_ids.shape == (2,)
|
||||
assert first.logits.shape == logits.shape
|
||||
assert torch.equal(first.token_ids, second.token_ids)
|
||||
|
||||
|
||||
def test_gumbel_config_warns_about_degenerate_generations(monkeypatch):
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.watermarking.logger.warning_once",
|
||||
lambda message, *, scope: messages.append(message),
|
||||
)
|
||||
|
||||
WatermarkConfig(key=42)
|
||||
|
||||
assert messages == [
|
||||
"Single-key Gumbel-max watermarking may increase the frequency of "
|
||||
"degenerate generations, including repetition loops."
|
||||
]
|
||||
|
||||
|
||||
def test_large_context_width_warns_but_is_allowed():
|
||||
config = WatermarkConfig(key=42, context_width=17)
|
||||
|
||||
with pytest.warns(UserWarning, match="reduce robustness to edits"):
|
||||
watermarker = create_watermarker(config)
|
||||
|
||||
assert watermarker.context_width == 17
|
||||
|
||||
|
||||
def test_sampling_params_can_disable_watermarking():
|
||||
assert SamplingParams().watermarking
|
||||
assert not SamplingParams.from_optional(watermarking=False).watermarking
|
||||
|
||||
|
||||
def test_gpu_sampler_warns_when_watermarking_is_enabled_for_greedy(monkeypatch):
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarking = SimpleNamespace(np=np.ones(1, dtype=bool))
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(Sampler, "add_request", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.logger.warning_once", messages.append
|
||||
)
|
||||
|
||||
sampler.add_request(0, 1, SamplingParams(temperature=0))
|
||||
sampler.add_request(0, 1, SamplingParams(temperature=1))
|
||||
sampler.add_request(0, 1, SamplingParams(temperature=0, watermarking=False))
|
||||
|
||||
assert messages == [
|
||||
(
|
||||
"Watermarking is enabled, but greedy decoding (temperature=0) cannot be "
|
||||
"watermarked. This request will use ordinary greedy sampling."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_gpu_sampler_respects_mixed_request_watermarking(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
def sample(self, logits, contexts, random_sample):
|
||||
return WatermarkSample(torch.tensor([7, 7]), logits + 10)
|
||||
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = StubWatermarker()
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.array([True, False]), gpu=torch.tensor([True, False])
|
||||
)
|
||||
sampler.sampling_states = SimpleNamespace(
|
||||
temperature=SimpleNamespace(np=np.ones(2), gpu=torch.ones(2)),
|
||||
seeds=SimpleNamespace(gpu=torch.zeros(2, dtype=torch.int64)),
|
||||
)
|
||||
sampler.use_fp64_gumbel = False
|
||||
sampler._get_contexts = lambda expanded_idx_mapping: torch.zeros(
|
||||
2, 1, dtype=torch.int64
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.gumbel_sample",
|
||||
lambda *args, **kwargs: torch.tensor([3, 4]),
|
||||
)
|
||||
logits = torch.zeros(2, 8)
|
||||
|
||||
sampled, output_logits = sampler._sample_random(
|
||||
logits,
|
||||
torch.tensor([0, 1]),
|
||||
np.array([0, 1]),
|
||||
torch.zeros(2, dtype=torch.int64),
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 4]))
|
||||
assert torch.equal(output_logits[0], torch.full((8,), 10.0))
|
||||
assert torch.equal(output_logits[1], logits[1])
|
||||
|
||||
|
||||
def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
|
||||
class StubWatermarker:
|
||||
context_width = 1
|
||||
|
||||
def sample(self, logits, contexts, random_sample):
|
||||
raise AssertionError("watermarker should not run for greedy requests")
|
||||
|
||||
sampler = object.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = StubWatermarker()
|
||||
sampler.watermarking = SimpleNamespace(
|
||||
np=np.array([True, True]), gpu=torch.tensor([True, True])
|
||||
)
|
||||
sampler.sampling_states = SimpleNamespace(
|
||||
temperature=SimpleNamespace(np=np.zeros(2), gpu=torch.zeros(2)),
|
||||
)
|
||||
expected = (torch.tensor([3, 4]), torch.zeros(2, 8))
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.Sampler._sample_random",
|
||||
lambda *args, **kwargs: expected,
|
||||
)
|
||||
|
||||
actual = sampler._sample_random(
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([0, 1]),
|
||||
np.array([0, 1]),
|
||||
torch.zeros(2, dtype=torch.int64),
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
assert actual is expected
|
||||
Reference in New Issue
Block a user